diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1dd8181425..62d5e3cfa9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,98 @@ env: GOTESTSUM_CMD: "gotestsum --format standard-verbose --debug --" GOTESTSUM_CMD_RAW: "gotestsum --format standard-verbose --debug --raw-command -- go tool test2json -t" + LCOW_ARTIFACT_PROJECT: "ContainerPlatform" + LCOW_ARTIFACT_FEED: "ContainerPlat-Dev" + LCOW_ARTIFACT_NAME: "azurelinux-uvm" + LCOW_ARTIFACT_VERSION: "0.0.13" + LINUX_BOOT_FILES_PATH: ${{ github.workspace }}/LinuxBootFiles + jobs: + create-linux-boot-files: + runs-on: [self-hosted, 1ES.Pool=containerplat-github-runner-pool-east-us-2, 1ES.ImageOverride=github-mms-ubuntu-22] + permissions: + id-token: write # This is required for OIDC login (azure/login) to succeed + contents: read # This is required for actions/checkout to succeed + steps: + - name: Checkout hcsshim + uses: actions/checkout@v4 + with: + show-progress: false + + - name: Azure OIDC Login + uses: azure/login@v2 + with: + client-id: "930a0428-2b45-4cf9-9afe-b81bde516504" + tenant-id: "72f988bf-86f1-41af-91ab-2d7cd011db47" + allow-no-subscriptions: true + + - name: Download artifact from feed + uses: azure/cli@v2 + with: + azcliversion: latest + inlineScript: | + az extension add --name azure-devops + export DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 + + az artifacts universal download \ + --organization "https://msazure.visualstudio.com/" \ + --project ${{ env.LCOW_ARTIFACT_PROJECT }} \ + --scope project \ + --feed ${{ env.LCOW_ARTIFACT_FEED }} \ + --name ${{ env.LCOW_ARTIFACT_NAME }} \ + --version ${{ env.LCOW_ARTIFACT_VERSION }} \ + --path ./downloaded_artifacts + + - name: Show downloaded lcow artifacts + run: find ./downloaded_artifacts -maxdepth 3 -ls + + - name: Create directory for storing linux boot files + run: mkdir -p ${{ env.LINUX_BOOT_FILES_PATH }}/ + + - name: Copy Linux kernel and rootfs tar files + run: | + mv ./downloaded_artifacts/LinuxBootFiles/kernel ${{ env.LINUX_BOOT_FILES_PATH }}/ + mv ./downloaded_artifacts/LinuxBootFiles/vmlinux ${{ env.LINUX_BOOT_FILES_PATH }}/ + mv ./downloaded_artifacts/rootfs-*.tar.gz . + + - name: Create a new rootfs.vhd and initrd.img containing the local Linux-GCS + run: | + # Update package on Ubuntu + sudo apt-get update + sudo apt-get install -y make gcc binutils linux-headers-generic \ + libarchive-tools btrfs-progs libseccomp-dev pkg-config cpio + + chmod a+x ${{ github.workspace }}/hack/catcpio.sh + + # Find the full file name for rootfs tar + ROOTFS_TAR=$(ls rootfs-*.tar.gz | head -n 1) + echo "The full file name is $ROOTFS_TAR" + + make clean + sudo make BASE=${{ github.workspace }}/$ROOTFS_TAR rootfs + + - name: Copy newly created rootfs.vhd and initrd.img + run: | + mv out/rootfs.vhd ${{ env.LINUX_BOOT_FILES_PATH }}/ + mv out/initrd.img ${{ env.LINUX_BOOT_FILES_PATH }}/ + + # This is a workaround to overcome the limitation of actions/upload-artifact@v4 used in later jobs. + # See https://github.com/actions/upload-artifact/tree/v4/?tab=readme-ov-file#permission-loss. + - name: Tar the files to preserve file permissions prior to upload + run: | + cd ${{ env.LINUX_BOOT_FILES_PATH }} + tar -cvf ../linux_boot_files.tar . + + # Upload the Linux boot files so that they can be used in the later jobs. + - name: Upload Linux boot files to artifact + uses: actions/upload-artifact@v4 + with: + name: linux_artifact + path: linux_boot_files.tar + if-no-files-found: error + overwrite: true + retention-days: 1 + lint: runs-on: "windows-2022" strategy: @@ -267,7 +358,7 @@ jobs: test-windows: name: test-windows (${{ matrix.name }}) - needs: [lint, protos, verify-vendor, go-gen] + needs: [lint, protos, verify-vendor, go-gen, create-linux-boot-files] runs-on: ${{ matrix.runner }} strategy: fail-fast: false @@ -344,6 +435,26 @@ jobs: ${{ env.GOTESTSUM_CMD_RAW }} ./containerd-shim-runhcs-v1.test.exe '-test.v' working-directory: test + - name: Create directory for Linux boot files + shell: pwsh + run: mkdir -p ${{ env.LINUX_BOOT_FILES_PATH }}/ + + # Download Linux kernel files and newly created rootfs containing the Linux-GCS under testing. + - name: Download boot files from artifact + uses: actions/download-artifact@v4 + with: + name: linux_artifact + + - name: Extract Linux boot files + shell: pwsh + run: | + tar -xvf linux_boot_files.tar -C ${{ env.LINUX_BOOT_FILES_PATH }}/ + + - name: Display downloaded Linux boot files + shell: pwsh + run: | + Get-ChildItem -Recurse -Force -Path ${{ env.LINUX_BOOT_FILES_PATH }}/ + - name: Build and run functional testing binary run: | ${{ env.GO_BUILD_TEST_CMD }} ./functional @@ -360,8 +471,8 @@ jobs: exit $LASTEXITCODE } - # Don't run Linux uVM (ie, nested virt) or LCOW integrity tests. Windows uVM tests will be run on 1ES runner pool. - $cmd = '${{ env.GOTESTSUM_CMD_RAW }} ./functional.test.exe -exclude=LCOW,LCOWIntegrity -test.timeout=1h -test.v -log-level=info' + # Don't run LCOW integrity tests. Windows/Linux uVM tests will be run on 1ES Github Runner Pool. + $cmd = '${{ env.GOTESTSUM_CMD_RAW }} ./functional.test.exe -linux-bootfiles=${{ env.LINUX_BOOT_FILES_PATH }} -exclude=LCOWIntegrity -test.timeout=1h -test.v -log-level=info' $cmd = $cmd -replace 'gotestsum', $gotestsum Write-Host "gotestsum command: $cmd" diff --git a/Makefile b/Makefile index 9a9f5b4014..0d1550a361 100644 --- a/Makefile +++ b/Makefile @@ -64,14 +64,14 @@ out/delta-snp.tar.gz: out/delta.tar.gz bin/internal/tools/snp-report boot/startu out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths Makefile @mkdir -p out rm -rf rootfs - mkdir -p rootfs/bin/ + mkdir -p rootfs/usr/bin/ mkdir -p rootfs/info/ cp bin/init rootfs/ - cp bin/vsockexec rootfs/bin/ - cp bin/cmd/gcs rootfs/bin/ - cp bin/cmd/gcstools rootfs/bin/ - cp bin/cmd/hooks/wait-paths rootfs/bin/ - for tool in $(GCS_TOOLS); do ln -s gcstools rootfs/bin/$$tool; done + cp bin/vsockexec rootfs/usr/bin/ + cp bin/cmd/gcs rootfs/usr/bin/ + cp bin/cmd/gcstools rootfs/usr/bin/ + cp bin/cmd/hooks/wait-paths rootfs/usr/bin/ + for tool in $(GCS_TOOLS); do ln -s gcstools rootfs/usr/bin/$$tool; done git -C $(SRCROOT) rev-parse HEAD > rootfs/info/gcs.commit && \ git -C $(SRCROOT) rev-parse --abbrev-ref HEAD > rootfs/info/gcs.branch && \ date --iso-8601=minute --utc > rootfs/info/tar.date @@ -81,6 +81,9 @@ out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/ho tar -zcf $@ -C rootfs . rm -rf rootfs + # # Create symlink from /usr/bin to /bin + # ln -s /usr/bin /bin + bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths bin/cmd/tar2ext4 bin/internal/tools/snp-report: @mkdir -p $(dir $@) GOOS=linux $(GO_BUILD) -o $@ $(SRCROOT)/$(@:bin/%=%)