Add code-coverage option to mach run, test-wpt and test-devtools (#39916)

Add a `--coverage` flag to `./mach build` and a `./mach coverage-report`
command to use `cargo llvm-cov` to generate a coverage report from the
raw profiles.

The workflow is: 
```
./mach build --coverage [--profile <cargo_profile>]
# Or test-wpt or test-devtools
./mach run --coverage [--profile <cargo_profile>]
# Note, that coverage-report needs to know the cargo build profile.
./mach coverage-report [--profile <cargo_profile>] [optional parameters for cargo-llvm-cov]
```
According to the LLVM documentation on source based coverage, the
optimization profile should not influence the accuracy of the coverage
profile, so we can gather coverage data from optimized builds.

Note that `./mach test-devtools --coverage` will not produce any
coverage profiles yet, since the test runner kills the servo binary,
which prevents writing the profile data at shutdown.
The same problem also affects `test-wpt` with `servodriver`, which will
be fixed by https://github.com/servo/servo/pull/40455.

Testing: Manually tested. A CI workflow to test wpt coverage will be
added in a follow-up PR.

---------

Signed-off-by: Jonathan Schwender <schwenderjonathan@gmail.com>

etc/coverage_workspace_wrapper.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+# Copyright 2025 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+"""
+This is a simple script intended to be used as a `RUSTC_WORKSPACE_WRAPPER`, adding the
+required flags for code coverage instrumentation, only to the local libraries in our
+workspace. This reduces the runtime overhead and the profile size significantly.
+We are not interested in the code coverage metrics of outside dependencies anyway.
+"""
+
+import os
+import sys
+
+
+def main():
+    # The first argument is the path to rustc, followed by its arguments
+    args = sys.argv[1:]
+    args += ["-Cinstrument-coverage=true", "--cfg=coverage"]
+
+    # Execute rustc with the modified arguments
+    os.execvp(args[0], args)
+
+
+if __name__ == "__main__":
+    main()

python/servo/build_commands.py

@@ -127,6 +127,19 @@ class MachCommands(CommandBase):
         host = servo.platform.host_triple()
         target_triple = self.target.triple()
 
+        if self.enable_code_coverage:
+            print("Building with code coverage instrumentation...")
+            # We don't want coverage for build-scripts and proc macros.
+            kwargs["target_override"] = target_triple
+            this_dir = pathlib.Path(os.path.dirname(__file__))
+            servo_root_dir = this_dir.parent.parent
+            coverage_workspace_wrapper = servo_root_dir.joinpath("etc/coverage_workspace_wrapper.py")
+            if not coverage_workspace_wrapper.exists():
+                print(
+                    f"Could not find rustc workspace wrapper script at expected location: {coverage_workspace_wrapper}"
+                )
+            env["RUSTC_WORKSPACE_WRAPPER"] = str(coverage_workspace_wrapper)
+
         if sanitizer.is_some():
             self.build_sanitizer_env(env, opts, kwargs, target_triple, sanitizer)
 

python/servo/command_base.py

@@ -29,7 +29,7 @@ from enum import Enum
 from glob import glob
 from os import path
 from subprocess import PIPE, CompletedProcess
-from typing import Any, Optional, Union, LiteralString, cast
+from typing import Any, Optional, Union, LiteralString, cast, List
 from collections.abc import Generator, Callable
 from xml.etree.ElementTree import XML
 
@@ -43,6 +43,7 @@ from servo.platform.build_target import AndroidTarget, BuildTarget, OpenHarmonyT
 from servo.util import download_file, get_default_cache_dir
 
 from python.servo.platform.build_target import SanitizerKind
+from python.servo.util import get_target_dir
 
 NIGHTLY_REPOSITORY_URL = "https://servo-builds2.s3.amazonaws.com/"
 ASAN_LEAK_SUPPRESSION_FILE = "support/suppressed_leaks_for_asan.txt"
@@ -90,6 +91,9 @@ class BuildType:
         else:
             return self.profile
 
+    def as_cargo_arg(self) -> List[str]:
+        return ["--profile", self.profile]
+
     def __eq__(self, other: object) -> bool:
         raise Exception("BUG: do not compare BuildType with ==")
 
@@ -258,6 +262,7 @@ class CommandBase(object):
     def __init__(self, context: Any) -> None:
         self.context = context
         self.enable_media = False
+        self.enable_code_coverage = False
         self.features = []
 
         # Default to native build target. This will later be overriden
@@ -336,7 +341,7 @@ class CommandBase(object):
 
     def get_binary_path(self, build_type: BuildType, sanitizer: SanitizerKind = SanitizerKind.NONE) -> str:
         base_path = util.get_target_dir()
-        if sanitizer.is_some() or self.target.is_cross_build():
+        if sanitizer.is_some() or self.target.is_cross_build() or self.enable_code_coverage:
             base_path = path.join(base_path, self.target.triple())
         binary_name = self.target.binary_name()
         binary_path = path.join(base_path, build_type.directory_name(), binary_name)
@@ -537,6 +542,7 @@ class CommandBase(object):
         build_type: bool = False,
         binary_selection: bool = False,
         package_configuration: bool = False,
+        coverage_report: bool = False,
     ) -> Callable:
         decorators = []
         if build_type or binary_selection:
@@ -658,6 +664,15 @@ class CommandBase(object):
                 CommandArgument("--bin", default=None, help="Launch with specific binary"),
                 CommandArgument("--nightly", "-n", default=None, help="Specify a YYYY-MM-DD nightly build to run"),
             ]
+        if build_configuration or binary_selection:
+            decorators += [
+                CommandArgument(
+                    "--coverage",
+                    default=False,
+                    action="store_true",
+                    help="Build / Run with code coverage instrumentation",
+                ),
+            ]
 
         def decorator_function(original_function: Callable) -> Callable:
             def configuration_decorator(self: CommandBase, *args: Any, **kwargs: Any) -> Callable:
@@ -680,6 +695,17 @@ class CommandBase(object):
                     self.features = kwargs.get("features", None) or []
                     self.enable_media = self.is_media_enabled(kwargs["media_stack"])
 
+                if build_configuration or binary_selection:
+                    self.enable_code_coverage = kwargs.pop("coverage", False)
+                    # In coverage report mode force-enable, so that the user doesn't need to
+                    # additionally specify `--coverage`.
+                    self.enable_code_coverage |= coverage_report
+                    if self.enable_code_coverage:
+                        target_dir = get_target_dir()
+                        # See `cargo llvm-cov show-env`. We only need the profile file environment variable
+                        # The other variables are only required when creating a coverage report.
+                        os.environ["LLVM_PROFILE_FILE"] = f"{target_dir}/servo-%p-%14m.profraw"
+
                 if binary_selection:
                     if "servo_binary" not in kwargs:
                         kwargs["servo_binary"] = (

python/servo/post_build_commands.py

@@ -13,7 +13,7 @@ import os.path as path
 import subprocess
 from subprocess import CompletedProcess
 from shutil import copy2
-from typing import Any
+from typing import Any, Optional, List
 
 import mozdebug
 
@@ -33,6 +33,8 @@ from servo.command_base import (
 )
 from servo.platform.build_target import is_android
 
+from python.servo.command_base import BuildType
+
 ANDROID_APP_NAME = "org.servo.servoshell"
 
 
@@ -192,6 +194,28 @@ class PostBuildCommands(CommandBase):
             else:
                 raise exception
 
+    @Command("coverage-report", description="Create Servo Code Coverage report.", category="post-build")
+    @CommandArgument("params", nargs="...", help="Command-line arguments to be passed through to cargo llvm-cov")
+    @CommandBase.common_command_arguments(binary_selection=True, build_type=True, coverage_report=True)
+    def coverage_report(self, build_type: BuildType, params: Optional[List[str]] = None, **kwargs: Any) -> int:
+        target_dir = servo.util.get_target_dir()
+        # See `cargo llvm-cov show-env`. We only export the values required at runtime.
+        os.environ["CARGO_LLVM_COV"] = "1"
+        os.environ["CARGO_LLVM_COV_SHOW_ENV"] = "1"
+        os.environ["CARGO_LLVM_COV_TARGET_DIR"] = target_dir
+        try:
+            cargo_llvm_cov_cmd = ["cargo", "llvm-cov", "report", "--target", self.target.triple()]
+            cargo_llvm_cov_cmd.extend(build_type.as_cargo_arg())
+            cargo_llvm_cov_cmd.extend(params or [])
+            subprocess.check_call(cargo_llvm_cov_cmd)
+        except subprocess.CalledProcessError as exception:
+            if exception.returncode < 0:
+                print(f"`cargo llvm-cov` was terminated by signal {-exception.returncode}")
+            else:
+                print(f"`cargo llvm-cov` exited with non-zero status {exception.returncode}")
+            return exception.returncode
+        return 0
+
     @Command("android-emulator", description="Run the Android emulator", category="post-build")
     @CommandArgument("args", nargs="...", help="Command-line arguments to be passed through to the emulator")
     def android_emulator(self, args: list[str] | None = None) -> int: