mirror of
https://github.com/servo/servo
synced 2026-04-25 17:15:48 +02:00
b675180fe7
some bencher jobs, specifically linux release profile jobs, measure runtime perf by running speedometer and dromaeo. doing this on GitHub-hosted runners is suboptimal, because GitHub-hosted runners are not under our control and their performance can vary wildly depending on what hosts we get and how busy they are. this patch depends on #39270 and #39271, and the new [ci3](https://ci3.servo.org/) and [ci4](https://ci4.servo.org/) servers deployed in servo/ci-runners#49. these servers provide a more controlled environment for benchmarking by using known hardware that runs one job at a time and no other work (servo/project#160), with some of the [techniques](https://github.com/servo/servo/wiki/Servo-Benchmarking-Report-(October-2024)#methodology) [we’ve](https://github.com/servo/servo/wiki/Servo-Benchmarking-Report-(November-2024)#methodology) [developed](https://github.com/servo/servo/wiki/Servo-Benchmarking-Report-%28December-2024%29#methodology) for accurate measurement: - [we disable CPU frequency boost and hyperthreading](364719f210...5c02999bbc (diff-e6f17b25776ca26c2880cc3a4e3b99a0642ea968a8d6214763cb6467cc1251cfR239-R256)) - [we pin guest CPUs to specific host CPUs](364719f210...5c02999bbc (diff-cdaac247bfd7d30f8c835083adab39c7ead8791802498285ea2ce9e023cc5f06R15-R26)) - [we isolate a subset of CPUs from all processes and scheduling interrupts](364719f210...5c02999bbc (diff-e6f17b25776ca26c2880cc3a4e3b99a0642ea968a8d6214763cb6467cc1251cfR97-R102)) - the bencher workflow does not take advantage of this yet, but it will in a later patch to use ci3 and ci4 for bencher jobs, we add them to the list of self-hosted runner servers, then make the bencher workflow try to find a servo-ubuntu2204-bench runner if speedometer and/or dromaeo have been requested. to avoid mixing data, we set the bencher “testbed” based on where the runner came from: - for GitHub-hosted runners, we continue to use “ubuntu-22.04” - for runners on ci3 or ci4, we use “self-hosted-image:[servo-ubuntu2204-bench](e911a23eff/profiles/servo-ubuntu2204-bench)” Testing: - before, always GitHub-hosted: [job run](https://github.com/servo/servo/actions/runs/18276911520/job/52032330450) → [report](https://bencher.dev/perf/servo/reports/86aa60cc-9d42-418f-a639-07b8604b30fb) - after, self-hosted: [job run](https://github.com/servo/servo/actions/runs/18404778338/job/52442477058) → [report](https://bencher.dev/perf/servo/reports/6feed0ac-655a-4e17-9351-41cba8d283b2) - after, GitHub-hosted: [job run](https://github.com/servo/servo/actions/runs/18404806546/job/52442697457) → [report](https://bencher.dev/perf/servo/reports/235a4ee0-340d-458b-9be4-953568b0923d) - there are also counterparts for other platforms in the workflow runs above Fixes: #39269 --------- Signed-off-by: Delan Azabani <dazabani@igalia.com>
93 lines
3.0 KiB
Python
Executable File
93 lines
3.0 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
|
|
from http.server import HTTPServer, SimpleHTTPRequestHandler
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import urllib
|
|
import json
|
|
import urllib.parse
|
|
|
|
|
|
# Port to run the HTTP server on for Dromaeo.
|
|
TEST_SERVER_PORT = 8192
|
|
|
|
|
|
# Run servo and print / parse the results for a specific Dromaeo module.
|
|
def run_servo(servo_exe, tests):
|
|
url = "http://localhost:{0}/dromaeo/web/index.html?{1}&automated&post_json".format(TEST_SERVER_PORT, tests)
|
|
args = [servo_exe, url, "-z", "-f"]
|
|
return subprocess.Popen(args)
|
|
|
|
|
|
# Print usage if command line args are incorrect
|
|
def print_usage():
|
|
print("USAGE: {0} tests servo_binary dromaeo_base_dir [<BMF JSON output> <Cargo profile>]".format(sys.argv[0]))
|
|
|
|
|
|
post_data = None
|
|
|
|
|
|
# Handle the POST at the end
|
|
class RequestHandler(SimpleHTTPRequestHandler):
|
|
def do_POST(self):
|
|
global post_data
|
|
self.send_response(200)
|
|
self.end_headers()
|
|
self.wfile.write(b"<HTML>POST OK.<BR><BR>")
|
|
length = int(self.headers.get('content-length'))
|
|
parameters = urllib.parse.parse_qs(self.rfile.read(length))
|
|
post_data = parameters[b'data']
|
|
|
|
def log_message(self, format, *args):
|
|
return
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) == 4 or len(sys.argv) == 6:
|
|
tests = sys.argv[1]
|
|
servo_exe = sys.argv[2]
|
|
base_dir = sys.argv[3]
|
|
bmf_output = ""
|
|
cargo_profile_prefix = ""
|
|
if len(sys.argv) == 6:
|
|
bmf_output = sys.argv[4]
|
|
cargo_profile_prefix = f"{sys.argv[5]}/"
|
|
os.chdir(base_dir)
|
|
|
|
# Ensure servo binary can be found
|
|
if not os.path.isfile(servo_exe):
|
|
print("Unable to find {0}. This script expects an existing build of Servo.".format(servo_exe))
|
|
sys.exit(1)
|
|
|
|
# Start the test server
|
|
server = HTTPServer(('', TEST_SERVER_PORT), RequestHandler)
|
|
|
|
print("Testing Dromaeo on Servo!")
|
|
proc = run_servo(servo_exe, tests)
|
|
while not post_data:
|
|
server.handle_request()
|
|
data = json.loads(post_data[0])
|
|
number = 0
|
|
length = 0
|
|
for test in data:
|
|
number = max(number, len(data[test]))
|
|
length = max(length, len(test))
|
|
print("\n Test{0} | Time".format(" " * (length - len("Test"))))
|
|
print("-{0}-|-{1}-".format("-" * length, "-" * number))
|
|
for test in data:
|
|
print(" {0}{1} | {2}".format(test, " " * (length - len(test)), data[test]))
|
|
if bmf_output:
|
|
output = dict()
|
|
for (k, v) in data.items():
|
|
output[f"{cargo_profile_prefix}Dromaeo/{k}"] = {'throughput': {'value': float(v)}}
|
|
with open(bmf_output, 'w', encoding='utf-8') as f:
|
|
json.dump(output, f, indent=4)
|
|
proc.kill()
|
|
else:
|
|
print_usage()
|