From 38d977343c2361f8442c97dc5128957fd23a00e5 Mon Sep 17 00:00:00 2001 From: Jonathan Schwender <55576758+jschwe@users.noreply.github.com> Date: Fri, 7 Nov 2025 15:38:24 +0100 Subject: [PATCH] Add code-coverage option to mach run, test-wpt and test-devtools (#39916) Add a `--coverage` flag to `./mach build` and a `./mach coverage-report` command to use `cargo llvm-cov` to generate a coverage report from the raw profiles. The workflow is: ``` ./mach build --coverage [--profile ] # Or test-wpt or test-devtools ./mach run --coverage [--profile ] # Note, that coverage-report needs to know the cargo build profile. ./mach coverage-report [--profile ] [optional parameters for cargo-llvm-cov] ``` According to the LLVM documentation on source based coverage, the optimization profile should not influence the accuracy of the coverage profile, so we can gather coverage data from optimized builds. Note that `./mach test-devtools --coverage` will not produce any coverage profiles yet, since the test runner kills the servo binary, which prevents writing the profile data at shutdown. The same problem also affects `test-wpt` with `servodriver`, which will be fixed by https://github.com/servo/servo/pull/40455. Testing: Manually tested. A CI workflow to test wpt coverage will be added in a follow-up PR. --------- Signed-off-by: Jonathan Schwender --- etc/coverage_workspace_wrapper.py | 33 +++++++++++++++++++++++++++++ python/servo/build_commands.py | 13 ++++++++++++ python/servo/command_base.py | 30 ++++++++++++++++++++++++-- python/servo/post_build_commands.py | 26 ++++++++++++++++++++++- 4 files changed, 99 insertions(+), 3 deletions(-) create mode 100755 etc/coverage_workspace_wrapper.py diff --git a/etc/coverage_workspace_wrapper.py b/etc/coverage_workspace_wrapper.py new file mode 100755 index 00000000000..f7eb80ba8c7 --- /dev/null +++ b/etc/coverage_workspace_wrapper.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +# Copyright 2025 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +""" +This is a simple script intended to be used as a `RUSTC_WORKSPACE_WRAPPER`, adding the +required flags for code coverage instrumentation, only to the local libraries in our +workspace. This reduces the runtime overhead and the profile size significantly. +We are not interested in the code coverage metrics of outside dependencies anyway. +""" + +import os +import sys + + +def main(): + # The first argument is the path to rustc, followed by its arguments + args = sys.argv[1:] + args += ["-Cinstrument-coverage=true", "--cfg=coverage"] + + # Execute rustc with the modified arguments + os.execvp(args[0], args) + + +if __name__ == "__main__": + main() diff --git a/python/servo/build_commands.py b/python/servo/build_commands.py index fae2ca569ce..4305e477c29 100644 --- a/python/servo/build_commands.py +++ b/python/servo/build_commands.py @@ -127,6 +127,19 @@ class MachCommands(CommandBase): host = servo.platform.host_triple() target_triple = self.target.triple() + if self.enable_code_coverage: + print("Building with code coverage instrumentation...") + # We don't want coverage for build-scripts and proc macros. + kwargs["target_override"] = target_triple + this_dir = pathlib.Path(os.path.dirname(__file__)) + servo_root_dir = this_dir.parent.parent + coverage_workspace_wrapper = servo_root_dir.joinpath("etc/coverage_workspace_wrapper.py") + if not coverage_workspace_wrapper.exists(): + print( + f"Could not find rustc workspace wrapper script at expected location: {coverage_workspace_wrapper}" + ) + env["RUSTC_WORKSPACE_WRAPPER"] = str(coverage_workspace_wrapper) + if sanitizer.is_some(): self.build_sanitizer_env(env, opts, kwargs, target_triple, sanitizer) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 60a531ce662..86187fe8cd4 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -29,7 +29,7 @@ from enum import Enum from glob import glob from os import path from subprocess import PIPE, CompletedProcess -from typing import Any, Optional, Union, LiteralString, cast +from typing import Any, Optional, Union, LiteralString, cast, List from collections.abc import Generator, Callable from xml.etree.ElementTree import XML @@ -43,6 +43,7 @@ from servo.platform.build_target import AndroidTarget, BuildTarget, OpenHarmonyT from servo.util import download_file, get_default_cache_dir from python.servo.platform.build_target import SanitizerKind +from python.servo.util import get_target_dir NIGHTLY_REPOSITORY_URL = "https://servo-builds2.s3.amazonaws.com/" ASAN_LEAK_SUPPRESSION_FILE = "support/suppressed_leaks_for_asan.txt" @@ -90,6 +91,9 @@ class BuildType: else: return self.profile + def as_cargo_arg(self) -> List[str]: + return ["--profile", self.profile] + def __eq__(self, other: object) -> bool: raise Exception("BUG: do not compare BuildType with ==") @@ -258,6 +262,7 @@ class CommandBase(object): def __init__(self, context: Any) -> None: self.context = context self.enable_media = False + self.enable_code_coverage = False self.features = [] # Default to native build target. This will later be overriden @@ -336,7 +341,7 @@ class CommandBase(object): def get_binary_path(self, build_type: BuildType, sanitizer: SanitizerKind = SanitizerKind.NONE) -> str: base_path = util.get_target_dir() - if sanitizer.is_some() or self.target.is_cross_build(): + if sanitizer.is_some() or self.target.is_cross_build() or self.enable_code_coverage: base_path = path.join(base_path, self.target.triple()) binary_name = self.target.binary_name() binary_path = path.join(base_path, build_type.directory_name(), binary_name) @@ -537,6 +542,7 @@ class CommandBase(object): build_type: bool = False, binary_selection: bool = False, package_configuration: bool = False, + coverage_report: bool = False, ) -> Callable: decorators = [] if build_type or binary_selection: @@ -658,6 +664,15 @@ class CommandBase(object): CommandArgument("--bin", default=None, help="Launch with specific binary"), CommandArgument("--nightly", "-n", default=None, help="Specify a YYYY-MM-DD nightly build to run"), ] + if build_configuration or binary_selection: + decorators += [ + CommandArgument( + "--coverage", + default=False, + action="store_true", + help="Build / Run with code coverage instrumentation", + ), + ] def decorator_function(original_function: Callable) -> Callable: def configuration_decorator(self: CommandBase, *args: Any, **kwargs: Any) -> Callable: @@ -680,6 +695,17 @@ class CommandBase(object): self.features = kwargs.get("features", None) or [] self.enable_media = self.is_media_enabled(kwargs["media_stack"]) + if build_configuration or binary_selection: + self.enable_code_coverage = kwargs.pop("coverage", False) + # In coverage report mode force-enable, so that the user doesn't need to + # additionally specify `--coverage`. + self.enable_code_coverage |= coverage_report + if self.enable_code_coverage: + target_dir = get_target_dir() + # See `cargo llvm-cov show-env`. We only need the profile file environment variable + # The other variables are only required when creating a coverage report. + os.environ["LLVM_PROFILE_FILE"] = f"{target_dir}/servo-%p-%14m.profraw" + if binary_selection: if "servo_binary" not in kwargs: kwargs["servo_binary"] = ( diff --git a/python/servo/post_build_commands.py b/python/servo/post_build_commands.py index f0ecb34ab45..ca97d8f9368 100644 --- a/python/servo/post_build_commands.py +++ b/python/servo/post_build_commands.py @@ -13,7 +13,7 @@ import os.path as path import subprocess from subprocess import CompletedProcess from shutil import copy2 -from typing import Any +from typing import Any, Optional, List import mozdebug @@ -33,6 +33,8 @@ from servo.command_base import ( ) from servo.platform.build_target import is_android +from python.servo.command_base import BuildType + ANDROID_APP_NAME = "org.servo.servoshell" @@ -192,6 +194,28 @@ class PostBuildCommands(CommandBase): else: raise exception + @Command("coverage-report", description="Create Servo Code Coverage report.", category="post-build") + @CommandArgument("params", nargs="...", help="Command-line arguments to be passed through to cargo llvm-cov") + @CommandBase.common_command_arguments(binary_selection=True, build_type=True, coverage_report=True) + def coverage_report(self, build_type: BuildType, params: Optional[List[str]] = None, **kwargs: Any) -> int: + target_dir = servo.util.get_target_dir() + # See `cargo llvm-cov show-env`. We only export the values required at runtime. + os.environ["CARGO_LLVM_COV"] = "1" + os.environ["CARGO_LLVM_COV_SHOW_ENV"] = "1" + os.environ["CARGO_LLVM_COV_TARGET_DIR"] = target_dir + try: + cargo_llvm_cov_cmd = ["cargo", "llvm-cov", "report", "--target", self.target.triple()] + cargo_llvm_cov_cmd.extend(build_type.as_cargo_arg()) + cargo_llvm_cov_cmd.extend(params or []) + subprocess.check_call(cargo_llvm_cov_cmd) + except subprocess.CalledProcessError as exception: + if exception.returncode < 0: + print(f"`cargo llvm-cov` was terminated by signal {-exception.returncode}") + else: + print(f"`cargo llvm-cov` exited with non-zero status {exception.returncode}") + return exception.returncode + return 0 + @Command("android-emulator", description="Run the Android emulator", category="post-build") @CommandArgument("args", nargs="...", help="Command-line arguments to be passed through to the emulator") def android_emulator(self, args: list[str] | None = None) -> int: