Add end-to-end test report generation (and minor refactoring)

.github/workflows/test.yml

@@ -81,6 +81,9 @@ jobs:
     - name: Make magic-nix-cache read-only by removing post-build-hook
       run: sed -i '/post-build-hook = magic-nix-cache-build-hook/d' $HOME/.config/nix/nix.conf
     - run: ./build test-e2e
+    - name: Add summary
+      run: cat test-output/test-report.md >> "$GITHUB_STEP_SUMMARY"
+      if: always()
 
   ocaml-tests:
     runs-on: ubuntu-latest

build

@@ -126,11 +126,11 @@ elif [ "$TARGET" == "test-e2e" ]; then
     --arg buildDisk false \
     --arg buildTest true
   "
-  echo "Building interactive e2e test runners."
+  echo "Building e2e test runners."
   (set -x; nix-build $test_flags)
   echo "Running e2e tests..."
-  (set -x; nix-build $test_flags -A tests --no-out-link)
-  echo "Done. All e2e tests passed."
+  (set -x; nix-build $test_flags -A tests -o test-output)
+  exit $(cat test-output/status)
 
 elif [ "$TARGET" == "all" ]; then
 

default.nix

@@ -156,10 +156,10 @@ with pkgs; stdenv.mkDerivation {
   ''
   # Tests
   + lib.optionalString buildTest ''
-    ln -s ${testComponents.tests.interactive-tests} $out/tests
+    mkdir -p $out/tests
+    ln -s ${testComponents.tests.interactive} $out/tests/interactive
+    ln -s ${testComponents.tests.tests} $out/tests/tests
   '';
 
-  passthru.tests = lib.optionalAttrs buildTest {
-    end-to-end = testComponents.tests.run-tests;
-  };
+  passthru.tests = testComponents.tests.run;
 }

testing/end-to-end/default.nix

@@ -5,16 +5,85 @@ let
     overlayPath = "/tmp/playos-test-disk-overlay.qcow2";
     # fileFilter is recursive, so tests can in theory be in subfolders
     testFiles = fileset.fileFilter (file: file.hasExt "nix") ./tests;
-    testPackages = map
-        (file: pkgs.callPackage file
-            (args // { inherit overlayPath; })
-        )
-        (fileset.toList testFiles);
-    testDeriv = pkgs.linkFarmFromDrvs "out" testPackages;
-    testInteractiveDeriv = pkgs.linkFarmFromDrvs "interactive"
-        (map (t: t.driverInteractive) testPackages);
+    testPackages = listToAttrs (map
+        (file: {
+            name = strings.removePrefix ((toString ./tests) + "/") (toString file);
+            value = pkgs.callPackage file (args // { inherit overlayPath; });
+        })
+        (fileset.toList testFiles)
+    );
+    testCases = driverAttr:
+        attrsets.mapAttrs'
+            (name: p: nameValuePair
+                ((strings.removeSuffix ".nix" name))
+                (p."${driverAttr}" + "/bin/nixos-test-driver")
+            )
+            testPackages;
+    testCasesInteractive = testCases "driverInteractive";
+    testCasesNormal = testCases "driver";
+    runAndSave = pkgs.writeShellScript "run-and-save" ''
+        set -euo pipefail
+        ansi2txt="${pkgs.colorized-logs}/bin/ansi2txt"
+        script="$1"
+        outDir="$2"
+        status=0
+        startTime=$(date +%s)
+        ($script 2>&1 | tee >($ansi2txt > $outDir/logs.txt)) || status=$?
+        endTime=$(date +%s)
+        echo -n "$status" > $outDir/status
+        echo -n "$((endTime - startTime))" > $outDir/duration
+    '';
+    genReport = pkgs.writers.writePython3 "gen-report"
+        { libraries = with pkgs.python3Packages; [ colorama ];
+          flakeIgnore = [ "E731" "E501" "E741" ];
+        }
+        (readFile ./gen-report.py);
 in
     {
-        run-tests = testDeriv;
-        interactive-tests = testInteractiveDeriv;
+        tests = pkgs.linkFarm "tests" testCasesNormal;
+        interactive = pkgs.linkFarm "interactive" testCasesInteractive;
+        run = pkgs.runCommand "run-e2e-tests"
+            { buildInputs = with pkgs; [ ncurses ]; }
+            ''
+            set -euo pipefail
+            mkdir -p $out
+
+            ${strings.toShellVar "tests" testCasesNormal}
+            tput="tput -T ansi"
+
+            isSuccess() {
+                local outDir="$1"
+                return $(cat $outDir/status)
+            }
+            print_bold() {
+                $tput bold; echo "$1"; $tput sgr0
+            }
+            text_green() {
+                $tput setaf 2; echo -n "$1"; $tput sgr0
+            }
+            text_red() {
+                $tput setaf 1; echo -n "$1"; $tput sgr0
+            }
+
+            numFailed=0
+
+            # Run tests
+            for testCase in "''${!tests[@]}"; do
+                outDir=$out/$testCase
+                mkdir -p $outDir
+                print_bold "===== Running e2e test $testCase ..."
+                ${runAndSave} "''${tests[$testCase]}" "$outDir"
+                if isSuccess $outDir; then
+                    print_bold "===== Test $testCase $(text_green 'succeeded ✓')"
+                else
+                    numFailed=$((numFailed+1))
+                    print_bold "===== Test $testCase $(text_red 'failed ✗')"
+                fi
+            done
+
+            ${genReport} $out
+            ${genReport} --format markdown $out > $out/test-report.md
+
+            echo -n "$numFailed" > $out/status
+        '';
     }

testing/end-to-end/gen-report.py

@@ -0,0 +1,134 @@
+import glob
+from pathlib import Path
+import argparse
+import datetime
+from colorama import Style, Fore
+import html
+import textwrap
+
+
+def process_test_case(test_name, test_case_dir):
+    with open(test_case_dir / "status", "r") as rf:
+        status = int(rf.read().strip())
+        success = status == 0
+
+    with open(test_case_dir / "duration", "r") as rf:
+        duration = int(rf.read().strip())
+
+    result = {
+        'name': test_name,
+        'filepath': '/testing/end-to-end/tests/' + test_name + ".nix",
+        'duration': duration,
+        'success': success,
+        'status': status,
+    }
+    if not success:
+        with open(test_case_dir / "logs.txt", "r") as rf:
+            logs = rf.readlines()
+            result['last_logs'] = "".join(logs[-100:])
+
+    return result
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('test_result_dir')
+    parser.add_argument('--format',
+                        choices=['terminal', 'markdown'],
+                        default='terminal')
+    return parser.parse_args()
+
+
+_id = lambda x: x
+
+
+def format_gen(full_report, bold_f=_id, ok_f=_id, fail_f=_id, log_f=_id):
+    header = """## End-to-end test result summary:"""
+    test_lines = []
+    for t in full_report['tests']:
+        if t['success']:
+            outcome_str = ok_f("OK")
+            maybe_logs = ""
+        else:
+            outcome_str = fail_f("FAIL")
+            maybe_logs = " " + log_f(t['last_logs'])
+        duration_str = str(datetime.timedelta(seconds=t['duration']))
+        test_str = f"- {bold_f(t['name'])}: {outcome_str} (duration: {duration_str})"
+        test_lines.append(test_str + maybe_logs)
+
+    counts = full_report['counts']
+    footer = "\n" + bold_f(f"Ran {counts['total']} tests, passed: {counts['passed']}, failed: {counts['failed']}")
+
+    return "\n".join([header] + test_lines + [footer])
+
+
+def format_markdown(full_report):
+    bold_f = lambda s: f"**{s}**"
+    ok_f = lambda s: f"{s} :heavy_check_mark:"
+    fail_f = lambda s: f"{s} :x:"
+
+    def log_f(logs):
+        lines = html.escape(logs).splitlines()
+        lines = [l if l.strip() else "<br/>" for l in lines]
+        log_str = "\n".join(lines)
+        return "\n" + textwrap.indent(f"""\
+<details>
+<summary>Last logs:</summary>
+<pre>
+{log_str}
+</pre>
+</details>""", 4 * ' ')
+
+    return format_gen(full_report, bold_f, ok_f, fail_f, log_f)
+
+
+def format_terminal(full_report):
+    bold_f = lambda s: Style.BRIGHT + s + Style.RESET_ALL
+    ok_f = lambda s: bold_f(f"{Fore.GREEN}{s} ✓{Fore.RESET}")
+    fail_f = lambda s: f"{Fore.RED}{s} ✗{Fore.RESET}"
+    log_f = lambda _: ""
+    return format_gen(full_report, bold_f, ok_f, fail_f, log_f)
+
+
+def print_report(full_report, format):
+    if format == "terminal":
+        s = format_terminal(full_report)
+    elif format == "markdown":
+        s = format_markdown(full_report)
+    else:
+        s = ""
+        raise RuntimeError(f"Unknown format: {format}")
+
+    print(s)
+
+
+def main():
+    opts = parse_args()
+    test_case_reports = []
+    failed = 0
+    passed = 0
+
+    glob_pat = opts.test_result_dir + "/**/status"
+    for status in glob.glob(glob_pat, recursive=True):
+        test_case_dir = Path(status).parent
+        test_name = str(test_case_dir.relative_to(opts.test_result_dir))
+        result = process_test_case(test_name, test_case_dir)
+        test_case_reports.append(result)
+        if result['success']:
+            passed += 1
+        else:
+            failed += 1
+
+    full_report = {
+        'counts': {
+            'total': passed + failed,
+            'failed': failed,
+            'passed': passed,
+        },
+        'tests': test_case_reports,
+    }
+    print_report(full_report, format=opts.format)
+
+
+if __name__ == "__main__":
+    main()

testing/end-to-end/tests/application/kiosk-persistence-helpers.py

@@ -3,6 +3,7 @@
 import requests
 import asyncio
 import pyppeteer # type: ignore
+import tempfile
 
 # Forward external `port` to 127.0.0.1:port and add firewall exception to allow
 # external access to internal services in PlayOS
@@ -45,12 +46,17 @@ async def retry_until_no_exception(func, retries=3, sleep=3.0):
 # due to nix sandboxing, network access is isolated, so
 # we run a minimal HTTP server for opening in the kiosk
 def run_stub_server(port):
-    with open("index.html", "w") as f:
+    d = tempfile.TemporaryDirectory()
+    with open(f"{d.name}/index.html", "w") as f:
         f.write("Hello world")
 
+    class Handler(http.server.SimpleHTTPRequestHandler):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, directory=d.name, **kwargs)
+
     server = http.server.HTTPServer(
         ("", port),
-        http.server.SimpleHTTPRequestHandler
+        Handler
     )
     print(f"Starting HTTP server on port {port}")
     # Running as a separate process to avoid GIL

testing/helpers/nixos-test-script-helpers.py

@@ -2,6 +2,7 @@
 import unittest
 from colorama import Fore, Style
 import re
+import sys
 
 # HACK: create writable cow disk overlay (same as in ./run-in-vm --disk)
 def create_overlay(disk, overlay_path):
@@ -13,15 +14,18 @@ def create_overlay(disk, overlay_path):
         ],
         check=True)
 
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
 class AbstractTestCheck(object):
     def __init__(self, check_kind, test_descr):
         self.check_kind = check_kind
         self.test_descr = test_descr
         self.test_c = unittest.TestCase()
 
     def print_descr(self, outcome=""):
-        print(f"{Style.BRIGHT}[{self.check_kind}] {self.test_descr}... {outcome}")
-        print(Style.RESET_ALL)
+        eprint(f"{Style.BRIGHT}[{self.check_kind}] {self.test_descr}... {outcome}")
+        eprint(Style.RESET_ALL)
 
     def print_ok(self):
         self.print_descr(outcome=f"{Fore.GREEN}OK!")
@@ -56,10 +60,10 @@ def wait_for_logs(vm, regex, unit=None, timeout=10):
     try:
         vm.wait_until_succeeds(full_cmd, timeout=timeout)
     except Exception as e:
-        print(f"wait_for_logs ({full_cmd}) failed after {timeout} seconds")
-        print("Last VM logs:\n")
+        eprint(f"wait_for_logs ({full_cmd}) failed after {timeout} seconds")
+        eprint("Last VM logs:\n")
         _, output = vm.execute(f"{journal_cmd} | tail -30")
-        print(output)
+        eprint(output)
         raise e