diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27148dfd..45a86ad7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/prep-build-env + - name: Make magic-nix-cache read-only by removing post-build-hook + run: sed -i '/post-build-hook = magic-nix-cache-build-hook/d' $HOME/.config/nix/nix.conf - run: NIXPKGS_ALLOW_UNFREE=1 nix-build -A driverInteractive ${{ matrix.file }} diff --git a/Readme.md b/Readme.md index 17241d8d..4ea7180a 100644 --- a/Readme.md +++ b/Readme.md @@ -105,11 +105,17 @@ Building a complete system takes time, so it is a good idea to test at the compo ### Automated Testing -There are 3 types of tests: +There are 4 types of automated tests: - unit tests (used in kiosk and controller) -- subcomponent integration tests (see below) -- end-to-end system level tests (see below) +- subcomponent integration tests +- end-to-end system level tests +- release validation tests + +All but the last are run on each commit via Github Actions. + +See subsections below for more details on each test type. + #### Subcomponent (integration) tests @@ -150,6 +156,35 @@ run 10x slower. If you see `failed to initialize KVM` in the console logs, it means there's a problem. See [this (outdated) Github issue](https://github.com/NixOS/nixpkgs/issues/124371#issue-900719073) for more details. + +#### Release validation (self-update) test + +The release validation test is used to perform the final _automated_ checks before +manually testing and officially publishing a release. + +Currently it tests only one critical path: the self-update scenario from an +earlier release (the 'base' system) to the current/upcoming release (the 'next' +system). + +The test does not alter the configuration of the base or next systems' in any way +(e.g. no test-instrumentation.nix extras, which are present in end-to-end +tests). + +To run the tests against the previous release, execute: + + nix-build testing/release-validation.nix + +To test against a specific base system (e.g. `2023.9.1-DISK`), add the flag +`--arg baseSystemVersion '"2023.9.1-DISK"'`. + +Compressed base system disk images are created for every tagged release using +`./build release-disk` and the [release](.github/workflows/release-tag.yml) +Github workflow. See the "Test disk" links in [release +summaries](https://github.com/dividat/playos/releases). + +For more details, see the comments in +[testing/release-validation.nix](testing/release-validation.nix). + ## Deployment Update bundles are hosted on Amazon S3. The script `bin/deploy-update` will handle signing and uploading of bundle. diff --git a/testing/release-validation.nix b/testing/release-validation.nix index ea7526fa..8f650fd0 100644 --- a/testing/release-validation.nix +++ b/testing/release-validation.nix @@ -24,13 +24,22 @@ # - execute "sendkey ctrl-shift-f1" (switch to TTY1) # - login with "root" let - baseS3URL = "https://dividat-playos-test-disks.s3.amazonaws.com/by-tag"; + # Note: we use HTTP instead of HTTPS, because pkgs.fetchurl fails + # when __structuredAttrs is enabled (due to mysterious OpenSSL/TLS errors) + # and also fails when __structuredAttrs is disabled (due to + # https://github.com/NixOS/nixpkgs/issues/177660). + # HTTP usage is fine since the output hash is fixed and verifies the download. + baseS3URL = "http://dividat-playos-test-disks.s3.amazonaws.com/by-tag"; # Generated via ./build release-disk and .github/workflows/release-tag.yml # See https://github.com/dividat/playos/releases diskImageURLs = { - "1.0.0-TEST" = { - url = "${baseS3URL}/playos-disk-1.0.0-TEST.img.zst"; - hash = "sha256-7cyStGfsxVyQ2ugkI9XRFnNrnPhd5QRf+oAQxLu3ovM="; + "2023.9.1-DISK" = { + url = "${baseS3URL}/playos-release-disk-2023.9.1-DISK.img.zst"; + hash = "sha256-eTyNcDkYSsMUsUHToZJ4tEKag9WSi8gA2SAihFFCqH0="; + }; + "2024.7.0-DISK" = { + url = "${baseS3URL}/playos-release-disk-2024.7.0-DISK.img.zst"; + hash = "sha256-vJDB99ICt0W1PmONikNY5wwIF7oQU388DzYRgPqkooY="; }; }; in @@ -48,9 +57,9 @@ in kioskUrlDomain ? "kiosk-server.local", # PlayOS system we are updating from - baseSystemVersion ? "1.0.0-TEST", + baseSystemVersion ? "2024.7.0-DISK", - # A downloadable URL containing a zstd compressed disk image + # A zstd-compressed PlayOS disk image baseSystemDiskImage ? (pkgs.fetchurl diskImageURLs.${baseSystemVersion}) .overrideAttrs { __structuredAttrs = true; @@ -150,20 +159,23 @@ pkgs.testers.runNixOSTest { "-monitor" "vc" ]; - enableOCR = true; - extraPythonPackages = ps: [ ps.colorama ps.types-colorama ps.requests ps.types-requests ps.tesserocr + ps.pillow + ps.types-pillow ]; testScript = {nodes}: '' ${builtins.readFile ./helpers/nixos-test-script-helpers.py} ${builtins.readFile ./end-to-end/tests/base/proxy-and-update-helpers.py} import tesserocr # type: ignore +import PIL.Image +import PIL.ImageEnhance +import PIL.ImageOps import tempfile import time import atexit @@ -191,13 +203,17 @@ extract_base_system_disk("${baseSystemDiskImage}", "${overlayPath}") playos.start(allow_reboot=True) sidekick.start() -# Less accurate, but much faster OCR than NixOS `get_screen_text`, -# which takes almost 20 seconds per call. -# Fails to identify white text on dark backgrounds. +# Faster OCR than NixOS `get_screen_text`, which takes almost 20 seconds per +# call. Fails to identify white text on dark backgrounds. def screenshot_and_ocr(vm): with tempfile.TemporaryDirectory() as d: vm.screenshot(d + "/screenshot.png") - return tesserocr.file_to_text(d + "/screenshot.png") + im = PIL.Image.open(d + "/screenshot.png") + im = PIL.ImageOps.grayscale(im) + im = PIL.ImageEnhance.Brightness(im).enhance(1.5) + im = PIL.ImageEnhance.Contrast(im).enhance(4.0) + return tesserocr.image_to_text(im) + def wait_until_passes(test, retries=10, sleep=1): while True: