From d9d526538ce2bfc157000caeade1ff4e4e7d2ece Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ignas=20Vy=C5=A1niauskas?= <i.vysniauskas@gmail.com>
Date: Tue, 22 Oct 2024 17:27:24 +0300
Subject: [PATCH 1/5] Use tempdir in e2e test to ensure it can be run anywhere

---
 .../tests/application/kiosk-persistence-helpers.py     | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/testing/end-to-end/tests/application/kiosk-persistence-helpers.py b/testing/end-to-end/tests/application/kiosk-persistence-helpers.py
index 09d72b08..04392882 100644
--- a/testing/end-to-end/tests/application/kiosk-persistence-helpers.py
+++ b/testing/end-to-end/tests/application/kiosk-persistence-helpers.py
@@ -3,6 +3,7 @@
 import requests
 import asyncio
 import pyppeteer # type: ignore
+import tempfile
 
 # Forward external `port` to 127.0.0.1:port and add firewall exception to allow
 # external access to internal services in PlayOS
@@ -45,12 +46,17 @@ async def retry_until_no_exception(func, retries=3, sleep=3.0):
 # due to nix sandboxing, network access is isolated, so
 # we run a minimal HTTP server for opening in the kiosk
 def run_stub_server(port):
-    with open("index.html", "w") as f:
+    d = tempfile.TemporaryDirectory()
+    with open(f"{d.name}/index.html", "w") as f:
         f.write("Hello world")
 
+    class Handler(http.server.SimpleHTTPRequestHandler):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, directory=d.name, **kwargs)
+
     server = http.server.HTTPServer(
         ("", port),
-        http.server.SimpleHTTPRequestHandler
+        Handler
     )
     print(f"Starting HTTP server on port {port}")
     # Running as a separate process to avoid GIL

From 340078f2af5638ca2d72fca1f827b5a15dfc7250 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ignas=20Vy=C5=A1niauskas?= <i.vysniauskas@gmail.com>
Date: Wed, 23 Oct 2024 17:50:03 +0300
Subject: [PATCH 2/5] Print test helper's logs to stderr for interleaving with
 VM logs

---
 testing/helpers/nixos-test-script-helpers.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/testing/helpers/nixos-test-script-helpers.py b/testing/helpers/nixos-test-script-helpers.py
index de255fb2..5395a6f9 100644
--- a/testing/helpers/nixos-test-script-helpers.py
+++ b/testing/helpers/nixos-test-script-helpers.py
@@ -2,6 +2,7 @@
 import unittest
 from colorama import Fore, Style
 import re
+import sys
 
 # HACK: create writable cow disk overlay (same as in ./run-in-vm --disk)
 def create_overlay(disk, overlay_path):
@@ -13,6 +14,9 @@ def create_overlay(disk, overlay_path):
         ],
         check=True)
 
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
 class AbstractTestCheck(object):
     def __init__(self, check_kind, test_descr):
         self.check_kind = check_kind
@@ -20,8 +24,8 @@ def __init__(self, check_kind, test_descr):
         self.test_c = unittest.TestCase()
 
     def print_descr(self, outcome=""):
-        print(f"{Style.BRIGHT}[{self.check_kind}] {self.test_descr}... {outcome}")
-        print(Style.RESET_ALL)
+        eprint(f"{Style.BRIGHT}[{self.check_kind}] {self.test_descr}... {outcome}")
+        eprint(Style.RESET_ALL)
 
     def print_ok(self):
         self.print_descr(outcome=f"{Fore.GREEN}OK!")
@@ -56,10 +60,10 @@ def wait_for_logs(vm, regex, unit=None, timeout=10):
     try:
         vm.wait_until_succeeds(full_cmd, timeout=timeout)
     except Exception as e:
-        print(f"wait_for_logs ({full_cmd}) failed after {timeout} seconds")
-        print("Last VM logs:\n")
+        eprint(f"wait_for_logs ({full_cmd}) failed after {timeout} seconds")
+        eprint("Last VM logs:\n")
         _, output = vm.execute(f"{journal_cmd} | tail -30")
-        print(output)
+        eprint(output)
         raise e
 
 

From 29cd7be124cc982228b360b6e4d41c1815edc7d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ignas=20Vy=C5=A1niauskas?= <i.vysniauskas@gmail.com>
Date: Tue, 22 Oct 2024 17:27:02 +0300
Subject: [PATCH 3/5] Refactor e2e test building and add report generation

---
 .github/workflows/test.yml       |   3 +
 build                            |   6 +-
 default.nix                      |   8 +-
 testing/end-to-end/default.nix   |  89 +++++++++++++++++++---
 testing/end-to-end/gen-report.py | 124 +++++++++++++++++++++++++++++++
 5 files changed, 213 insertions(+), 17 deletions(-)
 create mode 100644 testing/end-to-end/gen-report.py

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index b9c6e41d..8e46be62 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -81,6 +81,9 @@ jobs:
     - name: Make magic-nix-cache read-only by removing post-build-hook
       run: sed -i '/post-build-hook = magic-nix-cache-build-hook/d' $HOME/.config/nix/nix.conf
     - run: ./build test-e2e
+    - name: Add summary
+      run: cat test-output/test-report.md >> "$GITHUB_STEP_SUMMARY"
+      if: always()
 
   ocaml-tests:
     runs-on: ubuntu-latest
diff --git a/build b/build
index 6b7f8f35..83e3eaad 100755
--- a/build
+++ b/build
@@ -126,11 +126,11 @@ elif [ "$TARGET" == "test-e2e" ]; then
     --arg buildDisk false \
     --arg buildTest true
   "
-  echo "Building interactive e2e test runners."
+  echo "Building e2e test runners."
   (set -x; nix-build $test_flags)
   echo "Running e2e tests..."
-  (set -x; nix-build $test_flags -A tests --no-out-link)
-  echo "Done. All e2e tests passed."
+  (set -x; nix-build $test_flags -A tests -o test-output)
+  exit $(cat test-output/status)
 
 elif [ "$TARGET" == "all" ]; then
 
diff --git a/default.nix b/default.nix
index 137688b8..21970153 100644
--- a/default.nix
+++ b/default.nix
@@ -156,10 +156,10 @@ with pkgs; stdenv.mkDerivation {
   ''
   # Tests
   + lib.optionalString buildTest ''
-    ln -s ${testComponents.tests.interactive-tests} $out/tests
+    mkdir -p $out/tests
+    ln -s ${testComponents.tests.interactive} $out/tests/interactive
+    ln -s ${testComponents.tests.tests} $out/tests/tests
   '';
 
-  passthru.tests = lib.optionalAttrs buildTest {
-    end-to-end = testComponents.tests.run-tests;
-  };
+  passthru.tests = testComponents.tests.run;
 }
diff --git a/testing/end-to-end/default.nix b/testing/end-to-end/default.nix
index 8eba1dfa..00c98bf8 100644
--- a/testing/end-to-end/default.nix
+++ b/testing/end-to-end/default.nix
@@ -5,16 +5,85 @@ let
     overlayPath = "/tmp/playos-test-disk-overlay.qcow2";
     # fileFilter is recursive, so tests can in theory be in subfolders
     testFiles = fileset.fileFilter (file: file.hasExt "nix") ./tests;
-    testPackages = map
-        (file: pkgs.callPackage file
-            (args // { inherit overlayPath; })
-        )
-        (fileset.toList testFiles);
-    testDeriv = pkgs.linkFarmFromDrvs "out" testPackages;
-    testInteractiveDeriv = pkgs.linkFarmFromDrvs "interactive"
-        (map (t: t.driverInteractive) testPackages);
+    testPackages = listToAttrs (map
+        (file: {
+            name = strings.removePrefix ((toString ./tests) + "/") (toString file);
+            value = pkgs.callPackage file (args // { inherit overlayPath; });
+        })
+        (fileset.toList testFiles)
+    );
+    testCases = driverAttr:
+        attrsets.mapAttrs'
+            (name: p: nameValuePair
+                ((strings.removeSuffix ".nix" name))
+                (p."${driverAttr}" + "/bin/nixos-test-driver")
+            )
+            testPackages;
+    testCasesInteractive = testCases "driverInteractive";
+    testCasesNormal = testCases "driver";
+    runAndSave = pkgs.writeShellScript "run-and-save" ''
+        set -euo pipefail
+        ansi2txt="${pkgs.colorized-logs}/bin/ansi2txt"
+        script="$1"
+        outDir="$2"
+        status=0
+        startTime=$(date +%s)
+        ($script 2>&1 | tee >($ansi2txt > $outDir/logs.txt)) || status=$?
+        endTime=$(date +%s)
+        echo -n "$status" > $outDir/status
+        echo -n "$((endTime - startTime))" > $outDir/duration
+    '';
+    genReport = pkgs.writers.writePython3 "gen-report"
+        { libraries = with pkgs.python3Packages; [ colorama ];
+          flakeIgnore = [ "E731" "E501" ];
+        }
+        (readFile ./gen-report.py);
 in
     {
-        run-tests = testDeriv;
-        interactive-tests = testInteractiveDeriv;
+        tests = pkgs.linkFarm "tests" testCasesNormal;
+        interactive = pkgs.linkFarm "interactive" testCasesInteractive;
+        run = pkgs.runCommand "run-e2e-tests"
+            { buildInputs = with pkgs; [ ncurses ]; }
+            ''
+            set -euo pipefail
+            mkdir -p $out
+
+            ${strings.toShellVar "tests" testCasesNormal}
+            tput="tput -T ansi"
+
+            isSuccess() {
+                local outDir="$1"
+                return $(cat $outDir/status)
+            }
+            print_bold() {
+                $tput bold; echo "$1"; $tput sgr0
+            }
+            text_green() {
+                $tput setaf 2; echo -n "$1"; $tput sgr0
+            }
+            text_red() {
+                $tput setaf 1; echo -n "$1"; $tput sgr0
+            }
+
+            numFailed=0
+
+            # Run tests
+            for testCase in "''${!tests[@]}"; do
+                outDir=$out/$testCase
+                mkdir -p $outDir
+                print_bold "===== Running e2e test $testCase ..."
+                ${runAndSave} "''${tests[$testCase]}" "$outDir"
+                if isSuccess $outDir; then
+                    print_bold "===== Test $testCase $(text_green 'succeeded ✓')"
+                else
+                    numFailed=$((numFailed+1))
+                    print_bold "===== Test $testCase $(text_red 'failed ✗')"
+                fi
+            done
+
+            ${genReport} $out
+            ${genReport} --format markdown $out > $out/test-report.md
+
+            echo -n "$numFailed" > $out/status
+        '';
     }
diff --git a/testing/end-to-end/gen-report.py b/testing/end-to-end/gen-report.py
new file mode 100644
index 00000000..9f0bdec4
--- /dev/null
+++ b/testing/end-to-end/gen-report.py
@@ -0,0 +1,124 @@
+import glob
+from pathlib import Path
+import argparse
+import datetime
+from colorama import Style, Fore
+import html
+import textwrap
+
+
+def process_test_case(test_name, test_case_dir):
+    with open(test_case_dir / "status", "r") as rf:
+        status = int(rf.read().strip())
+        success = status == 0
+
+    with open(test_case_dir / "duration", "r") as rf:
+        duration = int(rf.read().strip())
+
+    result = {
+        'name': test_name,
+        'filepath': '/testing/end-to-end/tests/' + test_name + ".nix",
+        'duration': duration,
+        'success': success,
+        'status': status,
+    }
+    if not success:
+        with open(test_case_dir / "logs.txt", "r") as rf:
+            logs = rf.readlines()
+            result['last_logs'] = "".join(logs[-100:])
+
+    return result
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('test_result_dir')
+    parser.add_argument('--format',
+                        choices=['terminal', 'markdown'],
+                        default='terminal')
+    return parser.parse_args()
+
+
+_id = lambda x: x
+
+
+def format_gen(full_report, bold_f=_id, ok_f=_id, fail_f=_id, log_f=_id):
+    header = """## End-to-end test result summary:"""
+    test_lines = []
+    for t in full_report['tests']:
+        if t['success']:
+            outcome_str = ok_f("OK")
+            maybe_logs = ""
+        else:
+            outcome_str = fail_f("FAIL")
+            maybe_logs = " " + log_f(t['last_logs'])
+        duration_str = str(datetime.timedelta(seconds=t['duration']))
+        test_str = f"- {bold_f(t['name'])}: {outcome_str} (duration: {duration_str})"
+        test_lines.append(test_str + maybe_logs)
+
+    counts = full_report['counts']
+    footer = "\n" + bold_f(f"Ran {counts['total']} tests, passed: {counts['passed']}, failed: {counts['failed']}")
+
+    return "\n".join([header] + test_lines + [footer])
+
+
+def format_markdown(full_report):
+    bold_f = lambda s: f"**{s}**"
+    ok_f = lambda s: f"{s} :heavy_check_mark:"
+    fail_f = lambda s: f"{s} :x:"
+    log_f = lambda logs: f"""<details><summary>Last logs:</summary><pre>
+{textwrap.indent(html.escape(logs), 4 * ' ')}
+    </pre></details>"""
+    return format_gen(full_report, bold_f, ok_f, fail_f, log_f)
+
+
+def format_terminal(full_report):
+    bold_f = lambda s: Style.BRIGHT + s + Style.RESET_ALL
+    ok_f = lambda s: bold_f(f"{Fore.GREEN}{s} ✓{Fore.RESET}")
+    fail_f = lambda s: f"{Fore.RED}{s} ✗{Fore.RESET}"
+    log_f = lambda _: ""
+    return format_gen(full_report, bold_f, ok_f, fail_f, log_f)
+
+
+def print_report(full_report, format):
+    if format == "terminal":
+        s = format_terminal(full_report)
+    elif format == "markdown":
+        s = format_markdown(full_report)
+    else:
+        s = ""
+        raise RuntimeError(f"Unknown format: {format}")
+
+    print(s)
+
+
+def main():
+    opts = parse_args()
+    test_case_reports = []
+    failed = 0
+    passed = 0
+
+    glob_pat = opts.test_result_dir + "/**/status"
+    for status in glob.glob(glob_pat, recursive=True):
+        test_case_dir = Path(status).parent
+        test_name = str(test_case_dir.relative_to(opts.test_result_dir))
+        result = process_test_case(test_name, test_case_dir)
+        test_case_reports.append(result)
+        if result['success']:
+            passed += 1
+        else:
+            failed += 1
+
+    full_report = {
+        'counts': {
+            'total': passed + failed,
+            'failed': failed,
+            'passed': passed,
+        },
+        'tests': test_case_reports,
+    }
+    print_report(full_report, format=opts.format)
+
+
+if __name__ == "__main__":
+    main()

From f4b293b84c2003defaccb1345086a35691433f29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ignas=20Vy=C5=A1niauskas?= <i.vysniauskas@gmail.com>
Date: Wed, 23 Oct 2024 19:16:26 +0300
Subject: [PATCH 4/5] Make Github display the <pre> nicely

---
 testing/end-to-end/gen-report.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/testing/end-to-end/gen-report.py b/testing/end-to-end/gen-report.py
index 9f0bdec4..041e52ee 100644
--- a/testing/end-to-end/gen-report.py
+++ b/testing/end-to-end/gen-report.py
@@ -66,9 +66,17 @@ def format_markdown(full_report):
     bold_f = lambda s: f"**{s}**"
     ok_f = lambda s: f"{s} :heavy_check_mark:"
     fail_f = lambda s: f"{s} :x:"
-    log_f = lambda logs: f"""<details><summary>Last logs:</summary><pre>
-{textwrap.indent(html.escape(logs), 4 * ' ')}
-    </pre></details>"""
+    def log_f(logs):
+        lines = html.escape(logs).splitlines()
+        lines = [l if l.strip() else "<br/>" for l in lines]
+        log_str = "\n".join(lines)
+        return "\n" + textwrap.indent(f"""\
+<details>
+<summary>Last logs:</summary>
+<pre>
+{log_str}
+</pre>
+</details>""", 4 * ' ')
     return format_gen(full_report, bold_f, ok_f, fail_f, log_f)
 
 

From 2994c38b53cff249199f05071b92a9c42d7feec0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ignas=20Vy=C5=A1niauskas?= <i.vysniauskas@gmail.com>
Date: Wed, 23 Oct 2024 19:44:04 +0300
Subject: [PATCH 5/5] Make the linter happy

---
 testing/end-to-end/default.nix   | 2 +-
 testing/end-to-end/gen-report.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/testing/end-to-end/default.nix b/testing/end-to-end/default.nix
index 00c98bf8..b8173cce 100644
--- a/testing/end-to-end/default.nix
+++ b/testing/end-to-end/default.nix
@@ -35,7 +35,7 @@ let
     '';
     genReport = pkgs.writers.writePython3 "gen-report"
         { libraries = with pkgs.python3Packages; [ colorama ];
-          flakeIgnore = [ "E731" "E501" ];
+          flakeIgnore = [ "E731" "E501" "E741" ];
         }
         (readFile ./gen-report.py);
 in
diff --git a/testing/end-to-end/gen-report.py b/testing/end-to-end/gen-report.py
index 041e52ee..4c4adfe3 100644
--- a/testing/end-to-end/gen-report.py
+++ b/testing/end-to-end/gen-report.py
@@ -66,6 +66,7 @@ def format_markdown(full_report):
     bold_f = lambda s: f"**{s}**"
     ok_f = lambda s: f"{s} :heavy_check_mark:"
     fail_f = lambda s: f"{s} :x:"
+
     def log_f(logs):
         lines = html.escape(logs).splitlines()
         lines = [l if l.strip() else "<br/>" for l in lines]
@@ -77,6 +78,7 @@ def log_f(logs):
 {log_str}
 </pre>
 </details>""", 4 * ' ')
+
     return format_gen(full_report, bold_f, ok_f, fail_f, log_f)