Add option for generating coverage reports

Add a `--coverage` option in the `test` subcommand of the miri script.
This option, when set, will generate a coverage report after running the
tests.

`cargo-binutils` is needed as a dependency to generate the reports.

.github/workflows/ci.yml

@@ -62,7 +62,15 @@ jobs:
       - name: rustdoc
         run: RUSTDOCFLAGS="-Dwarnings" ./miri doc --document-private-items
 
-  # These jobs doesn't actually test anything, but they're only used to tell
+  coverage:
+    name: Coverage report
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/workflows/setup
+      - name: coverage
+        run: ./miri test --coverage
+  # These jobs don't actually test anything, but they're only used to tell
   # bors the build completed, as there is no practical way to detect when a
   # workflow is successful listening to webhooks only.
   #
@@ -92,7 +100,7 @@ jobs:
         contents: write
         # ... and create a PR.
         pull-requests: write
-    needs: [build, style]
+    needs: [build, style, coverage]
     if: github.event_name == 'schedule' && failure()
     steps:
       # Send a Zulip notification

ci/ci.sh

@@ -175,4 +175,4 @@ case $HOST_TARGET in
     echo "FATAL: unknown host target: $HOST_TARGET"
     exit 1
     ;;
-esac
+esac

miri-script/Cargo.toml

@@ -24,3 +24,4 @@ rustc_version = "0.4"
 dunce = "1.0.4"
 directories = "5"
 serde_json = "1"
+tempfile = "3.13.0"

miri-script/src/commands.rs

@@ -172,7 +172,13 @@ impl Command {
             Command::Install { flags } => Self::install(flags),
             Command::Build { flags } => Self::build(flags),
             Command::Check { flags } => Self::check(flags),
-            Command::Test { bless, flags, target } => Self::test(bless, flags, target),
+            Command::Test { bless, flags, target, coverage } =>
+                Self::test(
+                    bless,
+                    flags,
+                    target,
+                    coverage.then_some(crate::coverage::CoverageReport::new()?),
+                ),
             Command::Run { dep, verbose, many_seeds, target, edition, flags } =>
                 Self::run(dep, verbose, many_seeds, target, edition, flags),
             Command::Doc { flags } => Self::doc(flags),
@@ -458,9 +464,18 @@ impl Command {
         Ok(())
     }
 
-    fn test(bless: bool, mut flags: Vec<String>, target: Option<String>) -> Result<()> {
+    fn test(
+        bless: bool,
+        mut flags: Vec<String>,
+        target: Option<String>,
+        coverage: Option<crate::coverage::CoverageReport>,
+    ) -> Result<()> {
         let mut e = MiriEnv::new()?;
 
+        if let Some(report) = &coverage {
+            report.add_env_vars(&mut e)?;
+        }
+
         // Prepare a sysroot. (Also builds cargo-miri, which we need.)
         e.build_miri_sysroot(/* quiet */ false, target.as_deref())?;
 
@@ -479,6 +494,11 @@ impl Command {
         // Then test, and let caller control flags.
         // Only in root project as `cargo-miri` has no tests.
         e.test(".", &flags)?;
+
+        if let Some(coverage) = &coverage {
+            coverage.show_coverage_report(&e)?;
+        }
+
         Ok(())
     }
 

miri-script/src/coverage.rs

@@ -0,0 +1,89 @@
+use std::ffi::OsString;
+
+use anyhow::{Context, Result};
+use path_macro::path;
+use xshell::cmd;
+
+use crate::util::MiriEnv;
+
+/// CoverageReport can generate code coverage reports for miri.
+pub struct CoverageReport {
+    /// path is a temporary directory where coverage artifacts will be stored.
+    path: tempfile::TempDir,
+}
+
+impl CoverageReport {
+    /// new creates a new CoverageReport
+    ///
+    /// # Errors
+    ///
+    /// An error will be returned if a temporary directory could not be created.
+    pub fn new() -> Result<Self> {
+        Ok(Self { path: tempfile::TempDir::new()? })
+    }
+
+    /// add_env_vars will add the required environment variables to MiriEnv `e`.
+    pub fn add_env_vars(&self, e: &mut MiriEnv) -> Result<()> {
+        let mut rustflags = e.sh.var("RUSTFLAGS")?;
+        rustflags.push_str(" -C instrument-coverage");
+        e.sh.set_var("RUSTFLAGS", rustflags);
+
+        // Copy-pasting from: https://doc.rust-lang.org/rustc/instrument-coverage.html#instrumentation-based-code-coverage
+        // The format symbols below have the following meaning:
+        // - %p - The process ID.
+        // - %Nm - the instrumented binary’s signature:
+        //   The runtime creates a pool of N raw profiles, used for on-line
+        //   profile merging. The runtime takes care of selecting a raw profile
+        //   from the pool, locking it, and updating it before the program
+        //   exits. N must be between 1 and 9, and defaults to 1 if omitted
+        //   (with simply %m).
+        //
+        // Additionally the default for LLVM_PROFILE_FILE is default_%m_%p.profraw.
+        // So we just use the same value.
+        let file_template = self.path.path().join("default_%m_%p.profraw");
+        e.sh.set_var("LLVM_PROFILE_FILE", file_template);
+        Ok(())
+    }
+
+    /// show_coverage_report will print coverage information using the artifact
+    /// files in `self.path`.
+    pub fn show_coverage_report(&self, e: &MiriEnv) -> Result<()> {
+        let profraw_files: Vec<_> = self.profraw_files()?;
+
+        let profdata_bin = path!(e.libdir / ".." / "bin" / "llvm-profdata");
+
+        let merged_file = self.path.path().join("merged.profdata");
+
+        // Merge the profraw files
+        let profraw_files_cloned = profraw_files.iter();
+        cmd!(e.sh, "{profdata_bin} merge -sparse {profraw_files_cloned...} -o {merged_file}")
+            .quiet()
+            .run()?;
+
+        // Create the coverage report.
+        let cov_bin = path!(e.libdir / ".." / "bin" / "llvm-cov");
+        let miri_bin =
+            e.build_get_binary(".").context("failed to get filename of miri executable")?;
+        cmd!(
+            e.sh,
+            "{cov_bin} report --instr-profile={merged_file} --object {miri_bin} --sources src/"
+        )
+        .run()?;
+
+        Ok(())
+    }
+
+    /// profraw_files returns the profraw files in `self.path`.
+    ///
+    /// # Errors
+    ///
+    /// An error will be returned if `self.path` can't be read.
+    fn profraw_files(&self) -> Result<Vec<OsString>> {
+        Ok(std::fs::read_dir(&self.path)?
+            .filter_map(|r| r.ok())
+            .map(|e| e.path())
+            .filter(|p| p.extension().map(|e| e == "profraw").unwrap_or(false))
+            .map(|p| p.as_os_str().to_os_string())
+            .collect())
+    }
+}

miri-script/src/main.rs

@@ -2,6 +2,7 @@
 
 mod args;
 mod commands;
+mod coverage;
 mod util;
 
 use std::ops::Range;
@@ -34,6 +35,8 @@ pub enum Command {
         /// The cross-interpretation target.
         /// If none then the host is the target.
         target: Option<String>,
+        /// Produce coverage report if set.
+        coverage: bool,
         /// Flags that are passed through to the test harness.
         flags: Vec<String>,
     },
@@ -158,9 +161,12 @@ fn main() -> Result<()> {
             let mut target = None;
             let mut bless = false;
             let mut flags = Vec::new();
+            let mut coverage = false;
             loop {
                 if args.get_long_flag("bless")? {
                     bless = true;
+                } else if args.get_long_flag("coverage")? {
+                    coverage = true;
                 } else if let Some(val) = args.get_long_opt("target")? {
                     target = Some(val);
                 } else if let Some(flag) = args.get_other() {
@@ -169,7 +175,7 @@ fn main() -> Result<()> {
                     break;
                 }
             }
-            Command::Test { bless, flags, target }
+            Command::Test { bless, flags, target, coverage }
         }
         Some("run") => {
             let mut dep = false;

miri-script/src/util.rs

@@ -41,6 +41,8 @@ pub struct MiriEnv {
     pub sysroot: PathBuf,
     /// The shell we use.
     pub sh: Shell,
+    /// The library dir in the sysroot.
+    pub libdir: PathBuf,
 }
 
 impl MiriEnv {
@@ -96,7 +98,8 @@ impl MiriEnv {
         // so that Windows can find the DLLs.
         if cfg!(windows) {
             let old_path = sh.var("PATH")?;
-            let new_path = env::join_paths(iter::once(libdir).chain(env::split_paths(&old_path)))?;
+            let new_path =
+                env::join_paths(iter::once(libdir.clone()).chain(env::split_paths(&old_path)))?;
             sh.set_var("PATH", new_path);
         }
 
@@ -111,7 +114,7 @@ impl MiriEnv {
             std::process::exit(1);
         }
 
-        Ok(MiriEnv { miri_dir, toolchain, sh, sysroot, cargo_extra_flags })
+        Ok(MiriEnv { miri_dir, toolchain, sh, sysroot, cargo_extra_flags, libdir })
     }
 
     pub fn cargo_cmd(&self, crate_dir: impl AsRef<OsStr>, cmd: &str) -> Cmd<'_> {