test-and-deploy.yml

name: Test, Build and Deploy Images

env:
  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}

on:
    workflow_dispatch:
    pull_request:
    push:
    schedule:
    # Run every 6 days to keep our caches alive
    - cron: '0 0 */6 * *'

jobs:
  python-tests:
    runs-on: ubuntu-latest

    env:
      python-version: 3.11 # Our base image has Python 3.11

    steps:
    - name: Checkout
      uses: actions/checkout@v4

    - name: Set up Python ${{ env.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.python-version }}

    - name: Install dependencies
      run: |
        sudo apt-get install --no-install-recommends --assume-yes shellcheck parallel
        python -m pip install --upgrade pip
        pip install poetry

    - name: Run tests
      run: |
        .hooks/pre-push

    - name: Upload coverage
      uses: actions/upload-artifact@v4
      with:
        name: coverage
        path: htmlcov


  frontend-tests:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - uses: oven-sh/setup-bun@v1
        with:
          bun-version: 1.0.3

      - name: Bun install
        run: bun install --cwd ./core/frontend

      - name: Bun lint
        run: bun --cwd ./core/frontend lint

      - name: Bun build
        run: bun run --cwd ./core/frontend build

  deploy-docker-images:
    runs-on: ubuntu-latest
    needs: [frontend-tests, python-tests]
    strategy:
      fail-fast: false
      matrix:
        docker: [bootstrap, core]
        project: [blueos]
        platforms: ["linux/arm/v7,linux/arm64/v8,linux/amd64"]
    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@main
        with:
          android: true
          dotnet: true
          haskell: true
          large-packages: false
          docker-images: false
          swap-storage: true

      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 #Number of commits to fetch. 0 indicates all history for all branches and tags.
          submodules: recursive

      - name: Prepare
        id: prepare
        run: |
          # Deploy image with the name of the branch, if the build is a git tag, replace tag with the tag name.
          # If git tag matches semver, append latest tag to the push.
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          VERSION=${GITHUB_REF##*/}

          if [[ $GITHUB_REF == refs/tags/* ]]; then
            VERSION=${GITHUB_REF#refs/tags/}
          fi

          TAGS="--tag ${DOCKER_IMAGE}:${VERSION}"
          if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
            TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest"
          fi

          echo "docker_image=${DOCKER_IMAGE}" >> $GITHUB_OUTPUT
          echo "version=${VERSION}" >> $GITHUB_OUTPUT
          echo "buildx_args=\
            --build-arg GIT_DESCRIBE_TAGS=$(git describe --tags --long --always) \
            --build-arg VITE_APP_GIT_DESCRIBE=$(git describe --long --always --dirty --all) \
            --cache-from 'type=local,src=/tmp/.buildx-cache' \
            --cache-to 'type=local,dest=/tmp/.buildx-cache' \
            ${TAGS} \
            --file ${{ matrix.docker }}/Dockerfile ./${{ matrix.docker }}" >> $GITHUB_OUTPUT

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: all

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: latest

      - name: Cache Docker layers
        uses: actions/cache@v4
        id: cache
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }}
          restore-keys: |
            ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }}
            ${{ runner.os }}-buildx-${{ matrix.docker }}

      - name: Docker Buildx (build)
        run: |
          # Pull latest version of image to help with build speed
          for platform in $(echo ${{ matrix.platforms }} | tr ',' '\n'); do
            docker pull --platform ${platform} ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}:master || true
          done
          docker buildx build \
            --output "type=image,push=false" \
            --platform ${{ matrix.platforms }} \
            ${{ steps.prepare.outputs.buildx_args }}

      - name: Check core size
        if: ${{ matrix.docker == 'core' }}
        run: |
          # Check if the image size is lower than our limit
          docker image list
          IMAGE_ID=$(docker images -q ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }} | head -n 1)
          LIMIT_SIZE_MB=700
          IMAGE_SIZE_MB=$(( $(docker inspect $IMAGE_ID --format {{.Size}})/(2**20) ))
          echo "Core size is: $IMAGE_SIZE_MB MB"
          ((IMAGE_SIZE_MB < LIMIT_SIZE_MB))

      - name: Login to DockerHub
        if: success() && github.event_name != 'pull_request'
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Docker Buildx (push)
        if: success() && github.event_name != 'pull_request'
        run: |
          docker buildx build \
            --output "type=image,push=true" \
            --platform ${{ matrix.platforms }} \
            ${{ steps.prepare.outputs.buildx_args }}

      - name: Inspect image
        if: always() && github.event_name != 'pull_request'
        run: |
          docker buildx imagetools \
            inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }}

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/arm64/v8"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm64-v8.tar" \

      - name: Upload artifact arm64-v8
        uses: actions/upload-artifact@v4
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-arm64-v8
          path: '*arm64-v8.tar'

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/arm/v7"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm-v7.tar" \

      - name: Upload artifact arm-v7
        uses: actions/upload-artifact@v4
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-arm-v7
          path: '*arm-v7.tar'

      - name: Create image artifact
        if: success() && matrix.docker == 'core'
        run: |
          DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}
          GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA")
          docker buildx build \
            ${{ steps.prepare.outputs.buildx_args }} \
            --platform "linux/amd64"  \
            --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \
            --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-amd64.tar" \

      - name: Upload artifact amd64
        uses: actions/upload-artifact@v4
        if: success() && matrix.docker == 'core'
        with:
          name: BlueOS-core-docker-image-amd64
          path: '*amd64.tar'

      - name: Upload docker image for release
        uses: svenstaro/upload-release-action@v2
        if: startsWith(github.ref, 'refs/tags/') && success() && matrix.docker == 'core'
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: '*.tar'
          tag: ${{ github.ref }}
          overwrite: true
          prerelease: true
          file_glob: true

  deploy-raspberry-image:
    needs: deploy-docker-images
    if: github.event_name != 'pull_request' && github.repository_owner == 'bluerobotics'
    timeout-minutes: 180 # Detect if it gets into an infinite loop or some unexpected state
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner: blueos-ci
            platform: "linux/arm/v7"
            os: "bookworm"
            image: "raspios_lite_armhf/images/raspios_lite_armhf-2024-07-04/2024-07-04-raspios-bookworm-armhf-lite.img.xz"
          - runner: pi4-builder2
            platform: "linux/arm/v7"
            os: "bullseye"
            image: "raspios_lite_armhf/images/raspios_lite_armhf-2022-01-28/2022-01-28-raspios-bullseye-armhf-lite.zip"
          - runner: pi5-builder
            platform: "linux/arm64/v8"
            os: "bookworm"
            image: "raspios_lite_arm64/images/raspios_lite_arm64-2024-07-04/2024-07-04-raspios-bookworm-arm64-lite.img.xz"

    runs-on: ${{ matrix.runner }}
    # The runners for this job are:
    # - pi4-builders: A Raspberry Pi OS Bullseye (32-bit).
    # - pi5-builder: A Raspberry Pi OS Bookworm lite 64-bit.
    # For pi5-ci, install docker (curl -sSL https://get.docker.com/ | sh) and follow the instructions for setting up a new runner in
    # https://github.com/bluerobotics/BlueOS/settings/actions/runners/new

    steps:
      - name: Install git
        run: sudo apt install -y git

      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: recursive

      # We use our own pimod as upstream doesn't provide armv7 images
      - name: Pimod Build
        run: |
          VERSION=$GITHUB_REPOSITORY
          VERSION=${VERSION:-master}
          wget https://raw.githubusercontent.com//Nature40/pimod/master/pimod.sh && chmod +x pimod.sh
          docker run --rm --privileged \
            -v $PWD:/files \
            -e PATH=/pimod:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
            -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY \
            -e VERSION=$GITHUB_REF_NAME \
            -e BASE_IMAGE=${{ matrix.image }} \
            --workdir=/files \
            --platform ${{ matrix.platform }} nature40/pimod:latest pimod.sh deploy/pimod/blueos.Pifile

      - name: Add /boot additions
        run: |
          sudo apt-get update && sudo apt-get install -y parted kpartx
          # Create mount point if it doesn't exist
          sudo mkdir -p /mnt/piboot
          # Map the image file
          LOOP_DEVICE=$(sudo kpartx -avs deploy/pimod/blueos.img | sed -E 's/.*(loop[0-9]+)p[0-9]+.*/\1/g' | head -1)
          # Mount the boot partition
          sudo mount "/dev/mapper/${LOOP_DEVICE}p1" /mnt/piboot
          # Create ssh and userconf files
          sudo cp install/boards/config.toml /mnt/piboot/custom.toml
          sudo umount /mnt/piboot
          sudo kpartx -d deploy/pimod/blueos.img
          echo "Boot partition updated successfully."

      - name: Sanitize platform name
        run: echo "SANITIZED_PLATFORM=$(echo ${{ matrix.platform }} | tr '/' '-')" >> $GITHUB_ENV

      - name: Zip image
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          sudo apt install zip
          zip BlueOS-raspberry-${{ env.SANITIZED_PLATFORM }}-${{ matrix.os }}.zip deploy/pimod/blueos.img

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        timeout-minutes: 120
        with:
          name: BlueOS-raspberry-${{ env.GITHUB_REF_NAME }}${{ env.SANITIZED_PLATFORM }}-${{ matrix.os }}
          path: deploy/pimod/blueos.img
          if-no-files-found: error
          retention-days: 7

      - name: Set asset name
        run: |
          if [[ ${{ matrix.runner }} == 'pi5-builder' ]]; then
            echo "ASSET_NAME_SUFFIX=-pi5" >> $GITHUB_ENV
          elif [[ ${{ matrix.os }} == 'bullseye' ]]; then
            echo "ASSET_NAME_SUFFIX=-pi4" >> $GITHUB_ENV
          fi

      - name: Upload raspberry image for release
        uses: svenstaro/upload-release-action@v2
        if: startsWith(github.ref, 'refs/tags/')
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: BlueOS-raspberry-${{ env.SANITIZED_PLATFORM }}-${{ matrix.os }}.zip
          asset_name: BlueOS-raspberry-${{ env.SANITIZED_PLATFORM }}-${{ matrix.os }}${{ env.ASSET_NAME_SUFFIX }}.zip
          tag: ${{ github.ref }}
          overwrite: true
          prerelease: true

      # This is required because docker has root permissions, which means the runner is unable to clear this cache normally
      - name: Cleanup
        if: ${{ always() }}
        run: sudo rm -rf .cache