Merge pull request #219 from yfyf/backport-release-disk-2024.7.0

Backport release disk to 2024.7.0

.github/workflows/gen-release-summary.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -euo pipefail
+
+RELEASE_TAG="$1"
+
+# Find the previous "proper" release (i.e. not VALIDATION) tag.
+# Note: `--version-sort` incorrectly sorts semver pre-releases, but
+# these get filtered out later, so it does not matter.
+prev_tag="$(git tag \
+    | sort --version-sort \
+    | grep -B10000 "$RELEASE_TAG" \
+    | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' \
+    | head -n -1 \
+    | tail -1)" || echo ""
+
+if [[ -z "$prev_tag" ]]; then
+    echo "Error: failed to determine previous release tag, are you sure input tag $RELEASE_TAG exists?"
+    exit 1
+else
+    echo "Previous proper release tag is: $prev_tag" >&2
+fi
+
+echo -e "
+# Release $RELEASE_TAG
+
+## Artifacts
+
+- Test disk: [https://dividat-playos-test-disks.s3.amazonaws.com/by-tag/playos-release-disk-$RELEASE_TAG.img.zst](https://dividat-playos-test-disks.s3.amazonaws.com/by-tag/playos-release-disk-$RELEASE_TAG.img.zst)
+
+## Changelog
+
+"
+
+# print changelog since $prev_tag (exclusive)
+grep -E -B10000 "^# \[$prev_tag\]" ./Changelog.md \
+    | head -n -1 \
+    | sed -E 's/#+/\0##/'

.github/workflows/release-tag.yml

@@ -0,0 +1,72 @@
+name: Release Tag
+
+on:
+  push:
+    tags:
+      - "*"
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Ensure KVM is usable by nix-build
+        run: sudo chmod a+rwx /dev/kvm
+        shell: bash
+      - uses: cachix/install-nix-action@v18
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+          extra_nix_config: |
+            system-features = nixos-test benchmark big-parallel kvm
+      - uses: DeterminateSystems/magic-nix-cache-action@v8
+
+      - name: Make magic-nix-cache read-only by removing post-build-hook
+        run: sed -i '/post-build-hook = magic-nix-cache-build-hook/d' $HOME/.config/nix/nix.conf
+
+      - name: Validate tag
+        run: |
+          app_vsn="$(nix eval --raw -f application.nix 'version')"
+          if [ "$app_vsn" != "$GITHUB_REF_NAME" ]; then
+            echo "Git tag ($GITHUB_REF_NAME) does not match version in application.nix ($app_vsn), aborting!"
+            exit 1
+          fi
+
+      - name: Build release disk
+        run: ./build release-disk
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          aws-access-key-id:  ${{ secrets.TEST_DISKS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.TEST_DISKS_SECRET_ACCESS_KEY }}
+          aws-region: eu-central-1
+
+      - name: Publish to S3
+        run: ./.github/workflows/upload-test-disk.sh "$GITHUB_REF_NAME"
+
+      - name: Create Release summary
+        run: ./.github/workflows/gen-release-summary.sh "$GITHUB_REF_NAME" > ./release-notes.md
+
+      - name: Create Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          extra_args=""
+          if [[ "$GITHUB_REF_NAME" == *VALIDATION ]]; then
+            extra_args="--prerelease"
+          elif [[ "$GITHUB_REF_NAME" == *TEST ]]; then
+            extra_args="--draft"
+          fi
+
+          gh release create --verify-tag \
+            -F ./release-notes.md \
+            $extra_args \
+            "$GITHUB_REF_NAME"

.github/workflows/upload-test-disk.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+RELEASE_TAG="$1"
+
+set -euo pipefail
+set -x
+disk_path="$(readlink ./result/playos-release-disk-$RELEASE_TAG.img.zst)"
+target_url="s3://dividat-playos-test-disks/by-tag/playos-release-disk-$RELEASE_TAG.img.zst"
+echo "Uploading test disk to: $target_url"
+aws s3 cp "$disk_path" "$target_url"

application.nix

@@ -1,7 +1,7 @@
 rec {
     fullProductName = "Dividat PlayOS";
     safeProductName = "playos";
-    version = "2024.7.0";
+    version = "2024.7.0-DISK";
 
     greeting = label: ''
                                            _

build

@@ -106,6 +106,27 @@ elif [ "$TARGET" == "shed-key" ]; then
     --arg buildBundle false \
     --arg buildDisk false)
 
+# builds a disk to be used as a base image in ./testing/release-validation.nix
+elif [ "$TARGET" == "release-disk" ]; then
+
+  echo -e "
+Building release disk image for release validation tests.
+
+Note: requires around 30GiB of free space for storing the intermediate disk
+images. The final compressed disk image is much smaller (~4 GiB).
+"
+
+  (set -x; nix-build \
+      --arg kioskUrl "http://kiosk-server.local/" \
+      --arg updateUrl "http://update-server.local/" \
+      --arg buildVm false \
+      --arg buildInstaller false \
+      --arg buildBundle false \
+      --arg buildLive false \
+      --arg buildDisk false \
+      --arg buildReleaseDisk true
+  )
+
 elif [ "$TARGET" == "default" ]; then
 
   (set -x; nix-build)

default.nix

@@ -20,6 +20,7 @@ in
 , buildInstaller ? true
 , buildBundle ? true
 , buildDisk ? true
+, buildReleaseDisk ? false
 , buildLive ? true
 }:
 
@@ -31,8 +32,9 @@ let
     applicationOverlays = application.overlays;
   });
 
-  # lib.makeScope returns consistent set of packages that depend on each other (and is my new favorite nixpkgs trick)
-  components = with pkgs; lib.makeScope newScope (self: with self; {
+  # lib.makeScope returns consistent set of packages that depend on each other
+  mkComponents = { application, extraModules ? [ ], rescueSystemOpts ? {}, diskBuildEnabled ? buildDisk }:
+  (with pkgs; lib.makeScope newScope (self: with self; {
 
     inherit updateUrl deployUrl kioskUrl;
     inherit (application) version safeProductName fullProductName;
@@ -55,7 +57,10 @@ let
     updateCert = copyPathToStore updateCert;
 
     # System image as used in full installation
-    systemImage = callPackage ./system-image { application = application; };
+    systemImage = callPackage ./system-image {
+        application = application;
+        extraModules = extraModules;
+    };
 
     # USB live system
     live = callPackage ./live { application = application; };
@@ -86,8 +91,18 @@ let
     # Script for spinning up VMs
     run-in-vm = callPackage ./testing/run-in-vm {};
 
-  });
+  }));
+
+  components = mkComponents { inherit application; };
 
+  releaseDiskComponents = mkComponents {
+    inherit application;
+    extraModules = [ ./testing/system/passwordless-root.nix ];
+  };
+
+  releaseDisk = pkgs.callPackage ./testing/disk/release.nix {
+    inherit (releaseDiskComponents) install-playos;
+  };
 in
 
 with pkgs; stdenv.mkDerivation {
@@ -115,6 +130,12 @@ with pkgs; stdenv.mkDerivation {
   + lib.optionalString buildLive ''
     ln -s ${components.live}/iso/${components.safeProductName}-live-${components.version}.iso $out/${components.safeProductName}-live-${components.version}.iso
   ''
+  + lib.optionalString buildDisk ''
+    ln -s ${components.disk} $out/${components.safeProductName}-disk-${components.version}.img
+  ''
+  + lib.optionalString buildReleaseDisk ''
+    ln -s ${releaseDisk} $out/${components.safeProductName}-release-disk-${components.version}.img.zst
+  ''
   # Installer ISO image
   + lib.optionalString buildInstaller ''
     ln -s ${components.installer}/iso/${components.safeProductName}-installer-${components.version}.iso $out/${components.safeProductName}-installer-${components.version}.iso

installer/install-playos/install-playos.py

@@ -12,8 +12,8 @@
 import re
 from datetime import datetime
 
-PARTITION_SIZE_GB_SYSTEM = 10
-PARTITION_SIZE_GB_DATA = 5
+PARTITION_SIZE_GB_SYSTEM = 9
+PARTITION_SIZE_GB_DATA = 1
 
 GRUB_CFG = "@grubCfg@"
 GRUB_ENV = '/mnt/boot/grub/grubenv'

system-image/default.nix

@@ -1,5 +1,5 @@
 # Build an installable system image assuming a disk layout of a full A/B installation
-{pkgs, lib, updateCert, kioskUrl, playos-controller, application }:
+{pkgs, lib, updateCert, kioskUrl, playos-controller, application, extraModules ? [ ] }:
 with lib;
 let nixos = pkgs.importFromNixos ""; in
 (nixos {
@@ -13,7 +13,7 @@ let nixos = pkgs.importFromNixos ""; in
 
       # Application-specific module
       application.module
-    ];
+    ] ++ extraModules;
 
     # Storage
     fileSystems = {

testing/disk/release.nix

@@ -0,0 +1,49 @@
+# Similarly to testing/disk/default.nix, this builds a disk image containing
+# a full PlayOS installation, with these differences:
+# - It uses default system and boot partition sizes. Total disk size is ~20 GiB
+# - It produces a (sparsified) qcow2 image rather than a raw one. This reduces
+#   the image size to ~8GiB
+# - It compresses the final image using zstd to reduce disk usage.
+#   Final compressed file size is around ~4GiB.
+{ pkgs
+, lib
+, install-playos
+}:
+with pkgs;
+with lib;
+let
+  # all sizes in MiB
+  partSizes = {
+    boot = 525; # 525 MiB (matches install-playos default)
+    system = 1024 * 9;  # 9 GiB (install-playos default - 1GiB)
+    data = 400; # 400 MiB (same as testing/disk/default.nix)
+  };
+  diskSizeMiB = 8 + partSizes."boot" + partSizes."data" + (partSizes."system" * 2) + 1;
+in
+vmTools.runInLinuxVM (
+  runCommand "build-playos-release-disk"
+    {
+      buildInputs = [install-playos];
+
+      preVM = ''
+        diskImage=nixos.raw
+        truncate -s ${toString diskSizeMiB}MiB $diskImage
+      '';
+
+      postVM = ''
+        mkdir -p $out
+        ${pkgs.qemu}/bin/qemu-img convert -f raw -O qcow2 $diskImage $out/playos-disk.img
+        rm $diskImage
+        ${pkgs.zstd}/bin/zstd --rm -f $out/playos-disk.img -o $out/playos-disk.img.zst
+        diskImage=$out/playos-disk.img.zst
+      '';
+      memSize = 1024;
+    }
+    ''
+      # machine-id of development image is hardcoded.
+      install-playos \
+        --device /dev/vda \
+        --machine-id "f414cca8312548d29689ebf287fb67e0" \
+        --no-confirm
+    ''
+) + "/playos-disk.img.zst"

testing/system/passwordless-root.nix

@@ -0,0 +1,3 @@
+{
+  users.users.root.initialHashedPassword = "";
+}