From 8ff14cf6e5f98602ef686edfcf564d501f640fc8 Mon Sep 17 00:00:00 2001 From: Martin Pool Date: Sun, 21 Aug 2022 10:41:13 -0700 Subject: [PATCH] Write commands to outcomes.json and inspect them --- NEWS.md | 2 ++ src/cargo.rs | 59 ++++++++++++++++----------------- src/lab.rs | 8 ++--- src/outcome.rs | 7 +++- tests/cli.rs | 90 ++++++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 123 insertions(+), 43 deletions(-) diff --git a/NEWS.md b/NEWS.md index d5ee6a53..cb75201b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -8,6 +8,8 @@ - Improved: Write `mutants.out/outcomes.json` after the source-tree build and baseline tests so that it can be observed earlier on. +- Improved: `mutants.out/outcomes.json` includes the commands run. + ## 0.2.11 Released 2022-08-20 diff --git a/src/cargo.rs b/src/cargo.rs index 070076e0..04b138ca 100644 --- a/src/cargo.rs +++ b/src/cargo.rs @@ -44,17 +44,13 @@ impl CargoResult { /// Run one `cargo` subprocess, with a timeout, and with appropriate handling of interrupts. pub fn run_cargo( - cargo_args: &[String], + argv: &[String], in_dir: &Utf8Path, log_file: &mut LogFile, timeout: Duration, console: &Console, ) -> Result { let start = Instant::now(); - // When run as a Cargo subcommand, which is the usual/intended case, - // $CARGO tells us the right way to call back into it, so that we get - // the matching toolchain etc. - let cargo_bin = cargo_bin(); let mut env = PopenConfig::current_env(); // See @@ -62,13 +58,11 @@ pub fn run_cargo( // TODO: Maybe this should append instead of overwriting it...? env.push(("RUSTFLAGS".into(), "--cap-lints=allow".into())); - let mut argv: Vec = vec![cargo_bin]; - argv.extend(cargo_args.iter().cloned()); let message = format!("run {}", argv.join(" "),); log_file.message(&message); info!("{}", message); let mut child = Popen::create( - &argv, + argv, PopenConfig { stdin: Redirection::None, stdout: Redirection::File(log_file.open_append()?), @@ -116,13 +110,16 @@ pub fn run_cargo( /// Return the name of the cargo binary. fn cargo_bin() -> String { + // When run as a Cargo subcommand, which is the usual/intended case, + // $CARGO tells us the right way to call back into it, so that we get + // the matching toolchain etc. env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned()) } -/// Make up the argv for a cargo check/build/test invocation, not including -/// the cargo binary itself. -pub fn cargo_args(package_name: Option<&str>, phase: Phase, options: &Options) -> Vec { - let mut cargo_args = vec![phase.name().to_string()]; +/// Make up the argv for a cargo check/build/test invocation, including argv[0] as the +/// cargo binary itself. +pub fn cargo_argv(package_name: Option<&str>, phase: Phase, options: &Options) -> Vec { + let mut cargo_args = vec![cargo_bin(), phase.name().to_string()]; if phase == Phase::Check || phase == Phase::Build { cargo_args.push("--tests".to_string()); } @@ -233,23 +230,23 @@ pub fn locate_project(path: &Utf8Path) -> Result { mod test { use pretty_assertions::assert_eq; - use super::cargo_args; + use super::cargo_argv; use crate::{Options, Phase}; #[test] fn generate_cargo_args_for_baseline_with_default_options() { let options = Options::default(); assert_eq!( - cargo_args(None, Phase::Check, &options), - vec!["check", "--tests", "--workspace"] + cargo_argv(None, Phase::Check, &options)[1..], + ["check", "--tests", "--workspace"] ); assert_eq!( - cargo_args(None, Phase::Build, &options), - vec!["build", "--tests", "--workspace"] + cargo_argv(None, Phase::Build, &options)[1..], + ["build", "--tests", "--workspace"] ); assert_eq!( - cargo_args(None, Phase::Test, &options), - vec!["test", "--workspace"] + cargo_argv(None, Phase::Test, &options)[1..], + ["test", "--workspace"] ); } @@ -261,16 +258,16 @@ mod test { .additional_cargo_test_args .extend(["--lib", "--no-fail-fast"].iter().map(|s| s.to_string())); assert_eq!( - cargo_args(Some(package_name), Phase::Check, &options), - vec!["check", "--tests", "--package", package_name] + cargo_argv(Some(package_name), Phase::Check, &options)[1..], + ["check", "--tests", "--package", package_name] ); assert_eq!( - cargo_args(Some(package_name), Phase::Build, &options), - vec!["build", "--tests", "--package", package_name] + cargo_argv(Some(package_name), Phase::Build, &options)[1..], + ["build", "--tests", "--package", package_name] ); assert_eq!( - cargo_args(Some(package_name), Phase::Test, &options), - vec!["test", "--package", package_name, "--lib", "--no-fail-fast"] + cargo_argv(Some(package_name), Phase::Test, &options)[1..], + ["test", "--package", package_name, "--lib", "--no-fail-fast"] ); } @@ -284,16 +281,16 @@ mod test { .additional_cargo_args .extend(["--release".to_owned()]); assert_eq!( - cargo_args(None, Phase::Check, &options), - vec!["check", "--tests", "--workspace", "--release"] + cargo_argv(None, Phase::Check, &options)[1..], + ["check", "--tests", "--workspace", "--release"] ); assert_eq!( - cargo_args(None, Phase::Build, &options), - vec!["build", "--tests", "--workspace", "--release"] + cargo_argv(None, Phase::Build, &options)[1..], + ["build", "--tests", "--workspace", "--release"] ); assert_eq!( - cargo_args(None, Phase::Test, &options), - vec![ + cargo_argv(None, Phase::Test, &options)[1..], + [ "test", "--workspace", "--release", diff --git a/src/lab.rs b/src/lab.rs index e9dcedd8..3eb2c127 100644 --- a/src/lab.rs +++ b/src/lab.rs @@ -14,7 +14,7 @@ use rand::prelude::*; use serde::Serialize; use tracing::info; -use crate::cargo::{cargo_args, run_cargo}; +use crate::cargo::{cargo_argv, run_cargo}; use crate::console::{self, plural, Console}; use crate::mutate::Mutant; use crate::outcome::{LabOutcome, Outcome, Phase}; @@ -225,13 +225,13 @@ fn run_cargo_phases( for &phase in phases { let phase_start = Instant::now(); console.scenario_phase_started(phase); - let cargo_args = cargo_args(scenario.package_name(), phase, options); + let cargo_argv = cargo_argv(scenario.package_name(), phase, options); let timeout = match phase { Phase::Test => options.test_timeout(), _ => Duration::MAX, }; - let cargo_result = run_cargo(&cargo_args, in_dir, &mut log_file, timeout, console)?; - outcome.add_phase_result(phase, phase_start.elapsed(), cargo_result); + let cargo_result = run_cargo(&cargo_argv, in_dir, &mut log_file, timeout, console)?; + outcome.add_phase_result(phase, phase_start.elapsed(), cargo_result, &cargo_argv); console.scenario_phase_finished(phase); if (phase == Phase::Check && options.check_only) || !cargo_result.success() { break; diff --git a/src/outcome.rs b/src/outcome.rs index 5f3b258e..6a94774d 100644 --- a/src/outcome.rs +++ b/src/outcome.rs @@ -174,11 +174,13 @@ impl Outcome { phase: Phase, duration: Duration, cargo_result: CargoResult, + command: &[String], ) { self.phase_results.push(PhaseResult { phase, duration, cargo_result, + command: command.to_owned(), }); } @@ -283,6 +285,8 @@ pub struct PhaseResult { pub duration: Duration, /// Did it succeed? pub cargo_result: CargoResult, + /// What command was run, as an argv list. + pub command: Vec, } impl Serialize for PhaseResult { @@ -291,10 +295,11 @@ impl Serialize for PhaseResult { S: Serializer, { // custom serialize to omit inessential info - let mut ss = serializer.serialize_struct("PhaseResult", 3)?; + let mut ss = serializer.serialize_struct("PhaseResult", 4)?; ss.serialize_field("phase", &self.phase)?; ss.serialize_field("duration", &self.duration.as_secs_f64())?; ss.serialize_field("cargo_result", &self.cargo_result)?; + ss.serialize_field("command", &self.command)?; ss.end() } } diff --git a/tests/cli.rs b/tests/cli.rs index c8270386..7db7d6f3 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -390,7 +390,81 @@ fn workspace_tree_is_well_tested() { .arg(tmp_src_dir.path()) .assert() .success(); - // TODO: Check that --package arguments were passed, maybe by looking in the `outcomes.json` file. + // The outcomes.json has some summary data + let json_str = + fs::read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")).unwrap(); + println!("outcomes.json:\n{}", json_str); + let json: serde_json::Value = json_str.parse().unwrap(); + assert_eq!(json["total_mutants"].as_u64().unwrap(), 3); + assert_eq!(json["caught"].as_u64().unwrap(), 3); + assert_eq!(json["missed"].as_u64().unwrap(), 0); + assert_eq!(json["timeout"].as_u64().unwrap(), 0); + let outcomes = json["outcomes"].as_array().unwrap(); + + { + let sourcetree_json = outcomes[0].as_object().expect("outcomes[0] is an object"); + assert_eq!(sourcetree_json["scenario"].as_str().unwrap(), "SourceTree"); + assert_eq!(sourcetree_json["summary"], "Success"); + let sourcetree_phases = sourcetree_json["phase_results"].as_array().unwrap(); + assert_eq!(sourcetree_phases.len(), 1); + let sourcetree_command = sourcetree_phases[0]["command"].as_array().unwrap(); + assert_eq!(sourcetree_command[1..], ["build", "--tests", "--workspace"]); + } + + { + let baseline = outcomes[1].as_object().unwrap(); + assert_eq!(baseline["scenario"].as_str().unwrap(), "Baseline"); + assert_eq!(baseline["summary"], "Success"); + let baseline_phases = baseline["phase_results"].as_array().unwrap(); + assert_eq!(baseline_phases.len(), 2); + assert_eq!(baseline_phases[0]["cargo_result"], "Success"); + assert_eq!( + baseline_phases[0]["command"].as_array().unwrap()[1..], + ["build", "--tests", "--workspace"] + ); + assert_eq!(baseline_phases[1]["cargo_result"], "Success"); + assert_eq!( + baseline_phases[1]["command"].as_array().unwrap()[1..], + ["test", "--workspace"] + ); + } + + assert_eq!(outcomes.len(), 5); + for outcome in &outcomes[2..] { + let mutant = &outcome["scenario"]["Mutant"]; + let package_name = mutant["package"].as_str().unwrap(); + assert!(!package_name.is_empty()); + assert_eq!(outcome["summary"], "CaughtMutant"); + let mutant_phases = outcome["phase_results"].as_array().unwrap(); + assert_eq!(mutant_phases.len(), 2); + assert_eq!(mutant_phases[0]["cargo_result"], "Success"); + assert_eq!( + mutant_phases[0]["command"].as_array().unwrap()[1..], + ["build", "--tests", "--package", package_name] + ); + assert_eq!(mutant_phases[1]["cargo_result"], "Failure"); + assert_eq!( + mutant_phases[1]["command"].as_array().unwrap()[1..], + ["test", "--package", package_name], + ); + } + { + let baseline = json["outcomes"][1].as_object().unwrap(); + assert_eq!(baseline["scenario"].as_str().unwrap(), "Baseline"); + assert_eq!(baseline["summary"], "Success"); + let baseline_phases = baseline["phase_results"].as_array().unwrap(); + assert_eq!(baseline_phases.len(), 2); + assert_eq!(baseline_phases[0]["cargo_result"], "Success"); + assert_eq!( + baseline_phases[0]["command"].as_array().unwrap()[1..], + ["build", "--tests", "--workspace"] + ); + assert_eq!(baseline_phases[1]["cargo_result"], "Success"); + assert_eq!( + baseline_phases[1]["command"].as_array().unwrap()[1..], + ["test", "--workspace"] + ); + } } #[test] @@ -482,11 +556,14 @@ fn well_tested_tree_quiet() { insta::assert_snapshot!(stdout); true })); - // The format of outcomes.json is not pinned down yet, but it should exist. - assert!(tmp_src_dir - .path() - .join("mutants.out/outcomes.json") - .exists()); + let outcomes_json = + fs::read_to_string(tmp_src_dir.path().join("mutants.out/outcomes.json")).unwrap(); + println!("outcomes.json:\n{}", outcomes_json); + let outcomes: serde_json::Value = outcomes_json.parse().unwrap(); + assert_eq!(outcomes["total_mutants"], 15); + assert_eq!(outcomes["caught"], 15); + assert_eq!(outcomes["unviable"], 0); + assert_eq!(outcomes["missed"], 0); } #[test] @@ -504,7 +581,6 @@ fn well_tested_tree_finds_no_problems() { insta::assert_snapshot!(stdout); true })); - // The format of outcomes.json is not pinned down yet, but it should exist. assert!(tmp_src_dir .path() .join("mutants.out/outcomes.json")