From 4aa0f4da72e35d82468bb8d366cdc250b5d8195e Mon Sep 17 00:00:00 2001 From: Konstantin Andrikopoulos Date: Wed, 9 Oct 2024 02:02:42 +0200 Subject: [PATCH] Add option for generating coverage reports Add a `--coverage` option in the `test` subcommand of the miri script. This option, when set, will generate a coverage report after running the tests. `cargo-binutils` is needed as a dependency to generate the reports. --- .github/workflows/ci.yml | 12 ++++- ci/ci.sh | 2 +- miri-script/Cargo.lock | 43 ++++++++++++++++-- miri-script/Cargo.toml | 1 + miri-script/src/commands.rs | 24 +++++++++- miri-script/src/coverage.rs | 89 +++++++++++++++++++++++++++++++++++++ miri-script/src/main.rs | 8 +++- miri-script/src/util.rs | 7 ++- 8 files changed, 174 insertions(+), 12 deletions(-) create mode 100644 miri-script/src/coverage.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b0916f511..33e00f3db0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,15 @@ jobs: - name: rustdoc run: RUSTDOCFLAGS="-Dwarnings" ./miri doc --document-private-items - # These jobs doesn't actually test anything, but they're only used to tell + coverage: + name: Coverage report + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/workflows/setup + - name: coverage + run: ./miri test --coverage + # These jobs don't actually test anything, but they're only used to tell # bors the build completed, as there is no practical way to detect when a # workflow is successful listening to webhooks only. # @@ -92,7 +100,7 @@ jobs: contents: write # ... and create a PR. pull-requests: write - needs: [build, style] + needs: [build, style, coverage] if: github.event_name == 'schedule' && failure() steps: # Send a Zulip notification diff --git a/ci/ci.sh b/ci/ci.sh index 689bc6d46f..7dc2dfcd2e 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -175,4 +175,4 @@ case $HOST_TARGET in echo "FATAL: unknown host target: $HOST_TARGET" exit 1 ;; -esac +esac \ No newline at end of file diff --git a/miri-script/Cargo.lock b/miri-script/Cargo.lock index 146e613c24..8dad30df6d 100644 --- a/miri-script/Cargo.lock +++ b/miri-script/Cargo.lock @@ -63,6 +63,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + [[package]] name = "getrandom" version = "0.2.12" @@ -100,9 +106,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libredox" @@ -138,11 +144,18 @@ dependencies = [ "rustc_version", "serde_json", "shell-words", + "tempfile", "walkdir", "which", "xshell", ] +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + [[package]] name = "option-ext" version = "0.2.0" @@ -195,9 +208,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -276,6 +289,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "thiserror" version = "1.0.57" @@ -357,6 +383,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" diff --git a/miri-script/Cargo.toml b/miri-script/Cargo.toml index 23b9a62515..5b31d5a6ff 100644 --- a/miri-script/Cargo.toml +++ b/miri-script/Cargo.toml @@ -24,3 +24,4 @@ rustc_version = "0.4" dunce = "1.0.4" directories = "5" serde_json = "1" +tempfile = "3.13.0" diff --git a/miri-script/src/commands.rs b/miri-script/src/commands.rs index 36175c8dd2..9e1ae66236 100644 --- a/miri-script/src/commands.rs +++ b/miri-script/src/commands.rs @@ -172,7 +172,13 @@ impl Command { Command::Install { flags } => Self::install(flags), Command::Build { flags } => Self::build(flags), Command::Check { flags } => Self::check(flags), - Command::Test { bless, flags, target } => Self::test(bless, flags, target), + Command::Test { bless, flags, target, coverage } => + Self::test( + bless, + flags, + target, + coverage.then_some(crate::coverage::CoverageReport::new()?), + ), Command::Run { dep, verbose, many_seeds, target, edition, flags } => Self::run(dep, verbose, many_seeds, target, edition, flags), Command::Doc { flags } => Self::doc(flags), @@ -458,9 +464,18 @@ impl Command { Ok(()) } - fn test(bless: bool, mut flags: Vec, target: Option) -> Result<()> { + fn test( + bless: bool, + mut flags: Vec, + target: Option, + coverage: Option, + ) -> Result<()> { let mut e = MiriEnv::new()?; + if let Some(report) = &coverage { + report.add_env_vars(&mut e)?; + } + // Prepare a sysroot. (Also builds cargo-miri, which we need.) e.build_miri_sysroot(/* quiet */ false, target.as_deref())?; @@ -479,6 +494,11 @@ impl Command { // Then test, and let caller control flags. // Only in root project as `cargo-miri` has no tests. e.test(".", &flags)?; + + if let Some(coverage) = &coverage { + coverage.show_coverage_report(&e)?; + } + Ok(()) } diff --git a/miri-script/src/coverage.rs b/miri-script/src/coverage.rs new file mode 100644 index 0000000000..71954f2a70 --- /dev/null +++ b/miri-script/src/coverage.rs @@ -0,0 +1,89 @@ +use std::ffi::OsString; + +use anyhow::{Context, Result}; +use path_macro::path; +use xshell::cmd; + +use crate::util::MiriEnv; + +/// CoverageReport can generate code coverage reports for miri. +pub struct CoverageReport { + /// path is a temporary directory where coverage artifacts will be stored. + path: tempfile::TempDir, +} + +impl CoverageReport { + /// new creates a new CoverageReport + /// + /// # Errors + /// + /// An error will be returned if a temporary directory could not be created. + pub fn new() -> Result { + Ok(Self { path: tempfile::TempDir::new()? }) + } + + /// add_env_vars will add the required environment variables to MiriEnv `e`. + pub fn add_env_vars(&self, e: &mut MiriEnv) -> Result<()> { + let mut rustflags = e.sh.var("RUSTFLAGS")?; + rustflags.push_str(" -C instrument-coverage"); + e.sh.set_var("RUSTFLAGS", rustflags); + + // Copy-pasting from: https://doc.rust-lang.org/rustc/instrument-coverage.html#instrumentation-based-code-coverage + // The format symbols below have the following meaning: + // - %p - The process ID. + // - %Nm - the instrumented binary’s signature: + // The runtime creates a pool of N raw profiles, used for on-line + // profile merging. The runtime takes care of selecting a raw profile + // from the pool, locking it, and updating it before the program + // exits. N must be between 1 and 9, and defaults to 1 if omitted + // (with simply %m). + // + // Additionally the default for LLVM_PROFILE_FILE is default_%m_%p.profraw. + // So we just use the same value. + let file_template = self.path.path().join("default_%m_%p.profraw"); + e.sh.set_var("LLVM_PROFILE_FILE", file_template); + Ok(()) + } + + /// show_coverage_report will print coverage information using the artifact + /// files in `self.path`. + pub fn show_coverage_report(&self, e: &MiriEnv) -> Result<()> { + let profraw_files: Vec<_> = self.profraw_files()?; + + let profdata_bin = path!(e.libdir / ".." / "bin" / "llvm-profdata"); + + let merged_file = self.path.path().join("merged.profdata"); + + // Merge the profraw files + let profraw_files_cloned = profraw_files.iter(); + cmd!(e.sh, "{profdata_bin} merge -sparse {profraw_files_cloned...} -o {merged_file}") + .quiet() + .run()?; + + // Create the coverage report. + let cov_bin = path!(e.libdir / ".." / "bin" / "llvm-cov"); + let miri_bin = + e.build_get_binary(".").context("failed to get filename of miri executable")?; + cmd!( + e.sh, + "{cov_bin} report --instr-profile={merged_file} --object {miri_bin} --sources src/" + ) + .run()?; + + Ok(()) + } + + /// profraw_files returns the profraw files in `self.path`. + /// + /// # Errors + /// + /// An error will be returned if `self.path` can't be read. + fn profraw_files(&self) -> Result> { + Ok(std::fs::read_dir(&self.path)? + .filter_map(|r| r.ok()) + .map(|e| e.path()) + .filter(|p| p.extension().map(|e| e == "profraw").unwrap_or(false)) + .map(|p| p.as_os_str().to_os_string()) + .collect()) + } +} diff --git a/miri-script/src/main.rs b/miri-script/src/main.rs index 0620f3aaf0..a329f62790 100644 --- a/miri-script/src/main.rs +++ b/miri-script/src/main.rs @@ -2,6 +2,7 @@ mod args; mod commands; +mod coverage; mod util; use std::ops::Range; @@ -34,6 +35,8 @@ pub enum Command { /// The cross-interpretation target. /// If none then the host is the target. target: Option, + /// Produce coverage report if set. + coverage: bool, /// Flags that are passed through to the test harness. flags: Vec, }, @@ -158,9 +161,12 @@ fn main() -> Result<()> { let mut target = None; let mut bless = false; let mut flags = Vec::new(); + let mut coverage = false; loop { if args.get_long_flag("bless")? { bless = true; + } else if args.get_long_flag("coverage")? { + coverage = true; } else if let Some(val) = args.get_long_opt("target")? { target = Some(val); } else if let Some(flag) = args.get_other() { @@ -169,7 +175,7 @@ fn main() -> Result<()> { break; } } - Command::Test { bless, flags, target } + Command::Test { bless, flags, target, coverage } } Some("run") => { let mut dep = false; diff --git a/miri-script/src/util.rs b/miri-script/src/util.rs index f5a6a8188a..e6e85747d4 100644 --- a/miri-script/src/util.rs +++ b/miri-script/src/util.rs @@ -41,6 +41,8 @@ pub struct MiriEnv { pub sysroot: PathBuf, /// The shell we use. pub sh: Shell, + /// The library dir in the sysroot. + pub libdir: PathBuf, } impl MiriEnv { @@ -96,7 +98,8 @@ impl MiriEnv { // so that Windows can find the DLLs. if cfg!(windows) { let old_path = sh.var("PATH")?; - let new_path = env::join_paths(iter::once(libdir).chain(env::split_paths(&old_path)))?; + let new_path = + env::join_paths(iter::once(libdir.clone()).chain(env::split_paths(&old_path)))?; sh.set_var("PATH", new_path); } @@ -111,7 +114,7 @@ impl MiriEnv { std::process::exit(1); } - Ok(MiriEnv { miri_dir, toolchain, sh, sysroot, cargo_extra_flags }) + Ok(MiriEnv { miri_dir, toolchain, sh, sysroot, cargo_extra_flags, libdir }) } pub fn cargo_cmd(&self, crate_dir: impl AsRef, cmd: &str) -> Cmd<'_> {