add debug-mismatch tool

commands.md

@@ -18,6 +18,7 @@ $ solana-test-suite [OPTIONS] COMMAND [ARGS]...
 
 * `create-fixtures`: Create test fixtures from a directory of...
 * `debug-instr`
+* `debug-mismatches`: Run tests on a set of targets with a list...
 * `decode-protobuf`: Convert InstrContext messages to...
 * `exec-instr`: Execute InstrContext message(s) and print...
 * `instr-from-fixtures`: Extract InstrContext messages from fixtures.
@@ -64,6 +65,26 @@ $ solana-test-suite debug-instr [OPTIONS]
 * `-d, --debugger TEXT`: Debugger to use (gdb, rust-gdb)  [default: gdb]
 * `--help`: Show this message and exit.
 
+## `solana-test-suite debug-mismatches`
+
+Run tests on a set of targets with a list of FuzzCorp mismatch links.
+
+Note: each `.so` target filename must be unique.
+
+**Usage**:
+
+```console
+$ solana-test-suite debug-mismatches [OPTIONS]
+```
+
+**Options**:
+
+* `-s, --solana-target PATH`: Solana (or ground truth) shared object (.so) target file path  [default: /home/kbhargava/repos/solfuzz-agave/target/release/libsolfuzz_agave.so]
+* `-t, --target PATH`: Shared object (.so) target file paths (pairs with --keep-passing). Targets must have sol_compat_instr_execute_v1 defined  [default: /home/kbhargava/repos/firedancer/build/native/gcc/lib/libfd_exec_sol_compat.so]
+* `-o, --output-dir PATH`: Output directory for InstrContext messages  [default: debug_mismatch]
+* `-u, --repro-urls TEXT`: Comma-delimited list of FuzzCorp mismatch links
+* `--help`: Show this message and exit.
+
 ## `solana-test-suite decode-protobuf`
 
 Convert InstrContext messages to human-readable format.

src/test_suite/test_suite.py

@@ -4,6 +4,7 @@
 import ctypes
 from multiprocessing import Pool
 from pathlib import Path
+import subprocess
 from test_suite.constants import LOG_FILE_SEPARATOR_LENGTH
 from test_suite.fixture_utils import (
     create_fixture,
@@ -449,6 +450,8 @@ def run_tests(
     if verbose:
         print(f"Failed tests: {failed_tests}")
         print(f"Skipped tests: {skipped_tests}")
+    if failed != 0 and save_failures:
+        print("Failures tests are in: ", globals.output_dir / "failed_protobufs")
 
 
 @app.command(
@@ -505,5 +508,103 @@ def list_harness_types():
     print(f"export HARNESS_TYPE={HARNESS_LIST[0]}")
 
 
+@app.command(
+    help=f"""
+            Run tests on a set of targets with a list of FuzzCorp mismatch links.
+
+            Note: each `.so` target filename must be unique.
+            """
+)
+def debug_mismatches(
+    solana_shared_library: Path = typer.Option(
+        Path(os.getenv("SOLFUZZ_TARGET", "")),
+        "--solana-target",
+        "-s",
+        help="Solana (or ground truth) shared object (.so) target file path",
+    ),
+    shared_libraries: List[Path] = typer.Option(
+        [Path(os.getenv("FIREDANCER_TARGET", ""))],
+        "--target",
+        "-t",
+        help="Shared object (.so) target file paths (pairs with --keep-passing)."
+        f" Targets must have {globals.harness_ctx.fuzz_fn_name} defined",
+    ),
+    output_dir: Path = typer.Option(
+        Path("debug_mismatch"),
+        "--output-dir",
+        "-o",
+        help=f"Output directory for {globals.harness_ctx.context_type.__name__} messages",
+    ),
+    repro_urls: str = typer.Option(
+        "", "--repro-urls", "-u", help="Comma-delimited list of FuzzCorp mismatch links"
+    ),
+):
+    fuzzcorp_cookie = os.getenv("FUZZCORP_COOKIE")
+    repro_urls_list = repro_urls.split(",") if repro_urls else []
+
+    custom_data_urls = []
+    for url in repro_urls_list:
+        result = subprocess.run(
+            ["curl", "--cookie", f"s={fuzzcorp_cookie}", f"{url}.bash"],
+            capture_output=True,
+            text=True,
+        )
+        start_index = result.stdout.find("REPRO_CUSTOM_URL=")
+        end_index = result.stdout.find("\n", start_index)
+        custom_url = result.stdout[
+            start_index + len("REPRO_CUSTOM_URL=") + 1 : end_index - 1
+        ].strip()
+        custom_data_urls.append(custom_url)
+
+    globals.output_dir = output_dir
+
+    if globals.output_dir.exists():
+        shutil.rmtree(globals.output_dir)
+    globals.output_dir.mkdir(parents=True, exist_ok=True)
+
+    globals.inputs_dir = globals.output_dir / "inputs"
+
+    if globals.inputs_dir.exists():
+        shutil.rmtree(globals.inputs_dir)
+    globals.inputs_dir.mkdir(parents=True, exist_ok=True)
+
+    for url in custom_data_urls:
+        zip_name = url.split("/")[-1]
+        result = subprocess.run(
+            ["wget", "-q", url, "-O", f"{globals.output_dir}/{zip_name}"],
+            capture_output=True,
+            text=True,
+        )
+
+        result = subprocess.run(
+            ["unzip", f"{globals.output_dir}/{zip_name}", "-d", globals.output_dir],
+            capture_output=True,
+            text=True,
+        )
+
+        result = subprocess.run(
+            f"mv {globals.output_dir}/repro_custom/*ctx {globals.inputs_dir}",
+            shell=True,
+            capture_output=True,
+            text=True,
+        )
+
+    run_tests(
+        file_or_dir=globals.inputs_dir,
+        solana_shared_library=solana_shared_library,
+        shared_libraries=shared_libraries,
+        output_dir=globals.output_dir / "test_results",
+        num_processes=4,
+        randomize_output_buffer=False,
+        log_chunk_size=10000,
+        verbose=True,
+        consensus_mode=False,
+        failures_only=False,
+        save_failures=True,
+    )
+
+    print(f"output: {globals.output_dir}")
+
+
 if __name__ == "__main__":
     app()