Write commands to outcomes.json and inspect them

NEWS.md

@@ -8,6 +8,8 @@
 
 - Improved: Write `mutants.out/outcomes.json` after the source-tree build and baseline tests so that it can be observed earlier on.
 
+- Improved: `mutants.out/outcomes.json` includes the commands run.
+
 ## 0.2.11
 
 Released 2022-08-20

src/cargo.rs

@@ -44,31 +44,25 @@ impl CargoResult {
 
 /// Run one `cargo` subprocess, with a timeout, and with appropriate handling of interrupts.
 pub fn run_cargo(
-    cargo_args: &[String],
+    argv: &[String],
     in_dir: &Utf8Path,
     log_file: &mut LogFile,
     timeout: Duration,
     console: &Console,
 ) -> Result<CargoResult> {
     let start = Instant::now();
-    // When run as a Cargo subcommand, which is the usual/intended case,
-    // $CARGO tells us the right way to call back into it, so that we get
-    // the matching toolchain etc.
-    let cargo_bin = cargo_bin();
 
     let mut env = PopenConfig::current_env();
     // See <https://doc.rust-lang.org/cargo/reference/environment-variables.html>
     // <https://doc.rust-lang.org/rustc/lints/levels.html#capping-lints>
     // TODO: Maybe this should append instead of overwriting it...?
     env.push(("RUSTFLAGS".into(), "--cap-lints=allow".into()));
 
-    let mut argv: Vec<String> = vec![cargo_bin];
-    argv.extend(cargo_args.iter().cloned());
     let message = format!("run {}", argv.join(" "),);
     log_file.message(&message);
     info!("{}", message);
     let mut child = Popen::create(
-        &argv,
+        argv,
         PopenConfig {
             stdin: Redirection::None,
             stdout: Redirection::File(log_file.open_append()?),
@@ -116,13 +110,16 @@ pub fn run_cargo(
 
 /// Return the name of the cargo binary.
 fn cargo_bin() -> String {
+    // When run as a Cargo subcommand, which is the usual/intended case,
+    // $CARGO tells us the right way to call back into it, so that we get
+    // the matching toolchain etc.
     env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned())
 }
 
-/// Make up the argv for a cargo check/build/test invocation, not including
-/// the cargo binary itself.
-pub fn cargo_args(package_name: Option<&str>, phase: Phase, options: &Options) -> Vec<String> {
-    let mut cargo_args = vec![phase.name().to_string()];
+/// Make up the argv for a cargo check/build/test invocation, including argv[0] as the
+/// cargo binary itself.
+pub fn cargo_argv(package_name: Option<&str>, phase: Phase, options: &Options) -> Vec<String> {
+    let mut cargo_args = vec![cargo_bin(), phase.name().to_string()];
     if phase == Phase::Check || phase == Phase::Build {
         cargo_args.push("--tests".to_string());
     }
@@ -233,23 +230,23 @@ pub fn locate_project(path: &Utf8Path) -> Result<Utf8PathBuf> {
 mod test {
     use pretty_assertions::assert_eq;
 
-    use super::cargo_args;
+    use super::cargo_argv;
     use crate::{Options, Phase};
 
     #[test]
     fn generate_cargo_args_for_baseline_with_default_options() {
         let options = Options::default();
         assert_eq!(
-            cargo_args(None, Phase::Check, &options),
-            vec!["check", "--tests", "--workspace"]
+            cargo_argv(None, Phase::Check, &options)[1..],
+            ["check", "--tests", "--workspace"]
         );
         assert_eq!(
-            cargo_args(None, Phase::Build, &options),
-            vec!["build", "--tests", "--workspace"]
+            cargo_argv(None, Phase::Build, &options)[1..],
+            ["build", "--tests", "--workspace"]
         );
         assert_eq!(
-            cargo_args(None, Phase::Test, &options),
-            vec!["test", "--workspace"]
+            cargo_argv(None, Phase::Test, &options)[1..],
+            ["test", "--workspace"]
         );
     }
 
@@ -261,16 +258,16 @@ mod test {
             .additional_cargo_test_args
             .extend(["--lib", "--no-fail-fast"].iter().map(|s| s.to_string()));
         assert_eq!(
-            cargo_args(Some(package_name), Phase::Check, &options),
-            vec!["check", "--tests", "--package", package_name]
+            cargo_argv(Some(package_name), Phase::Check, &options)[1..],
+            ["check", "--tests", "--package", package_name]
         );
         assert_eq!(
-            cargo_args(Some(package_name), Phase::Build, &options),
-            vec!["build", "--tests", "--package", package_name]
+            cargo_argv(Some(package_name), Phase::Build, &options)[1..],
+            ["build", "--tests", "--package", package_name]
         );
         assert_eq!(
-            cargo_args(Some(package_name), Phase::Test, &options),
-            vec!["test", "--package", package_name, "--lib", "--no-fail-fast"]
+            cargo_argv(Some(package_name), Phase::Test, &options)[1..],
+            ["test", "--package", package_name, "--lib", "--no-fail-fast"]
         );
     }
 
@@ -284,16 +281,16 @@ mod test {
             .additional_cargo_args
             .extend(["--release".to_owned()]);
         assert_eq!(
-            cargo_args(None, Phase::Check, &options),
-            vec!["check", "--tests", "--workspace", "--release"]
+            cargo_argv(None, Phase::Check, &options)[1..],
+            ["check", "--tests", "--workspace", "--release"]
         );
         assert_eq!(
-            cargo_args(None, Phase::Build, &options),
-            vec!["build", "--tests", "--workspace", "--release"]
+            cargo_argv(None, Phase::Build, &options)[1..],
+            ["build", "--tests", "--workspace", "--release"]
         );
         assert_eq!(
-            cargo_args(None, Phase::Test, &options),
-            vec![
+            cargo_argv(None, Phase::Test, &options)[1..],
+            [
                 "test",
                 "--workspace",
                 "--release",

src/lab.rs

@@ -14,7 +14,7 @@ use rand::prelude::*;
 use serde::Serialize;
 use tracing::info;
 
-use crate::cargo::{cargo_args, run_cargo};
+use crate::cargo::{cargo_argv, run_cargo};
 use crate::console::{self, plural, Console};
 use crate::mutate::Mutant;
 use crate::outcome::{LabOutcome, Outcome, Phase};
@@ -225,13 +225,13 @@ fn run_cargo_phases(
     for &phase in phases {
         let phase_start = Instant::now();
         console.scenario_phase_started(phase);
-        let cargo_args = cargo_args(scenario.package_name(), phase, options);
+        let cargo_argv = cargo_argv(scenario.package_name(), phase, options);
         let timeout = match phase {
             Phase::Test => options.test_timeout(),
             _ => Duration::MAX,
         };
-        let cargo_result = run_cargo(&cargo_args, in_dir, &mut log_file, timeout, console)?;
-        outcome.add_phase_result(phase, phase_start.elapsed(), cargo_result);
+        let cargo_result = run_cargo(&cargo_argv, in_dir, &mut log_file, timeout, console)?;
+        outcome.add_phase_result(phase, phase_start.elapsed(), cargo_result, &cargo_argv);
         console.scenario_phase_finished(phase);
         if (phase == Phase::Check && options.check_only) || !cargo_result.success() {
             break;

src/outcome.rs

@@ -174,11 +174,13 @@ impl Outcome {
         phase: Phase,
         duration: Duration,
         cargo_result: CargoResult,
+        command: &[String],
     ) {
         self.phase_results.push(PhaseResult {
             phase,
             duration,
             cargo_result,
+            command: command.to_owned(),
         });
     }
 
@@ -283,6 +285,8 @@ pub struct PhaseResult {
     pub duration: Duration,
     /// Did it succeed?
     pub cargo_result: CargoResult,
+    /// What command was run, as an argv list.
+    pub command: Vec<String>,
 }
 
 impl Serialize for PhaseResult {
@@ -291,10 +295,11 @@ impl Serialize for PhaseResult {
         S: Serializer,
     {
         // custom serialize to omit inessential info
-        let mut ss = serializer.serialize_struct("PhaseResult", 3)?;
+        let mut ss = serializer.serialize_struct("PhaseResult", 4)?;
         ss.serialize_field("phase", &self.phase)?;
         ss.serialize_field("duration", &self.duration.as_secs_f64())?;
         ss.serialize_field("cargo_result", &self.cargo_result)?;
+        ss.serialize_field("command", &self.command)?;
         ss.end()
     }
 }

tests/cli.rs

@@ -390,7 +390,81 @@ fn workspace_tree_is_well_tested() {
         .arg(tmp_src_dir.path())
         .assert()
         .success();
-    // TODO: Check that --package arguments were passed, maybe by looking in the `outcomes.json` file.
+    // The outcomes.json has some summary data
+    let json_str =
+        fs::read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")).unwrap();
+    println!("outcomes.json:\n{}", json_str);
+    let json: serde_json::Value = json_str.parse().unwrap();
+    assert_eq!(json["total_mutants"].as_u64().unwrap(), 3);
+    assert_eq!(json["caught"].as_u64().unwrap(), 3);
+    assert_eq!(json["missed"].as_u64().unwrap(), 0);
+    assert_eq!(json["timeout"].as_u64().unwrap(), 0);
+    let outcomes = json["outcomes"].as_array().unwrap();
+
+    {
+        let sourcetree_json = outcomes[0].as_object().expect("outcomes[0] is an object");
+        assert_eq!(sourcetree_json["scenario"].as_str().unwrap(), "SourceTree");
+        assert_eq!(sourcetree_json["summary"], "Success");
+        let sourcetree_phases = sourcetree_json["phase_results"].as_array().unwrap();
+        assert_eq!(sourcetree_phases.len(), 1);
+        let sourcetree_command = sourcetree_phases[0]["command"].as_array().unwrap();
+        assert_eq!(sourcetree_command[1..], ["build", "--tests", "--workspace"]);
+    }
+
+    {
+        let baseline = outcomes[1].as_object().unwrap();
+        assert_eq!(baseline["scenario"].as_str().unwrap(), "Baseline");
+        assert_eq!(baseline["summary"], "Success");
+        let baseline_phases = baseline["phase_results"].as_array().unwrap();
+        assert_eq!(baseline_phases.len(), 2);
+        assert_eq!(baseline_phases[0]["cargo_result"], "Success");
+        assert_eq!(
+            baseline_phases[0]["command"].as_array().unwrap()[1..],
+            ["build", "--tests", "--workspace"]
+        );
+        assert_eq!(baseline_phases[1]["cargo_result"], "Success");
+        assert_eq!(
+            baseline_phases[1]["command"].as_array().unwrap()[1..],
+            ["test", "--workspace"]
+        );
+    }
+
+    assert_eq!(outcomes.len(), 5);
+    for outcome in &outcomes[2..] {
+        let mutant = &outcome["scenario"]["Mutant"];
+        let package_name = mutant["package"].as_str().unwrap();
+        assert!(!package_name.is_empty());
+        assert_eq!(outcome["summary"], "CaughtMutant");
+        let mutant_phases = outcome["phase_results"].as_array().unwrap();
+        assert_eq!(mutant_phases.len(), 2);
+        assert_eq!(mutant_phases[0]["cargo_result"], "Success");
+        assert_eq!(
+            mutant_phases[0]["command"].as_array().unwrap()[1..],
+            ["build", "--tests", "--package", package_name]
+        );
+        assert_eq!(mutant_phases[1]["cargo_result"], "Failure");
+        assert_eq!(
+            mutant_phases[1]["command"].as_array().unwrap()[1..],
+            ["test", "--package", package_name],
+        );
+    }
+    {
+        let baseline = json["outcomes"][1].as_object().unwrap();
+        assert_eq!(baseline["scenario"].as_str().unwrap(), "Baseline");
+        assert_eq!(baseline["summary"], "Success");
+        let baseline_phases = baseline["phase_results"].as_array().unwrap();
+        assert_eq!(baseline_phases.len(), 2);
+        assert_eq!(baseline_phases[0]["cargo_result"], "Success");
+        assert_eq!(
+            baseline_phases[0]["command"].as_array().unwrap()[1..],
+            ["build", "--tests", "--workspace"]
+        );
+        assert_eq!(baseline_phases[1]["cargo_result"], "Success");
+        assert_eq!(
+            baseline_phases[1]["command"].as_array().unwrap()[1..],
+            ["test", "--workspace"]
+        );
+    }
 }
 
 #[test]
@@ -482,11 +556,14 @@ fn well_tested_tree_quiet() {
             insta::assert_snapshot!(stdout);
             true
         }));
-    // The format of outcomes.json is not pinned down yet, but it should exist.
-    assert!(tmp_src_dir
-        .path()
-        .join("mutants.out/outcomes.json")
-        .exists());
+    let outcomes_json =
+        fs::read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")).unwrap();
+    println!("outcomes.json:\n{}", outcomes_json);
+    let outcomes: serde_json::Value = outcomes_json.parse().unwrap();
+    assert_eq!(outcomes["total_mutants"], 15);
+    assert_eq!(outcomes["caught"], 15);
+    assert_eq!(outcomes["unviable"], 0);
+    assert_eq!(outcomes["missed"], 0);
 }
 
 #[test]
@@ -504,7 +581,6 @@ fn well_tested_tree_finds_no_problems() {
             insta::assert_snapshot!(stdout);
             true
         }));
-    // The format of outcomes.json is not pinned down yet, but it should exist.
     assert!(tmp_src_dir
         .path()
         .join("mutants.out/outcomes.json")