From 95f12b0f9c5a1222c8b85448b1453210bb4f8536 Mon Sep 17 00:00:00 2001 From: wuhuizuo Date: Wed, 8 Jan 2025 18:26:10 +0800 Subject: [PATCH] feat(ticdc): add dockerfiles for builder image for new CDC component (#525) Signed-off-by: wuhuizuo --------- Signed-off-by: wuhuizuo --- .../pull-cd-builder-images-centos7.yaml | 168 +++++++++++-- .github/workflows/pull-cd-builder-images.yaml | 165 +++++++++++-- .../workflows/pull-cd-builder-images2.yaml | 2 +- .../release-cd-builder-images-centos7.yaml | 164 +++++++++++-- .../workflows/release-cd-builder-images.yaml | 170 +++++++++++-- Justfile | 5 +- .../ng-monitoring/skaffold-centos7.yaml | 72 ++++++ .../cd/builders/ng-monitoring/skaffold.yaml | 55 +++++ .../cd/builders/pd/skaffold-centos7.yaml | 72 ++++++ dockerfiles/cd/builders/pd/skaffold.yaml | 55 +++++ dockerfiles/cd/builders/skaffold-centos7.yaml | 228 ------------------ dockerfiles/cd/builders/skaffold.yaml | 204 ---------------- dockerfiles/cd/builders/tem/skaffold.yaml | 55 +++++ dockerfiles/cd/builders/ticdc/Dockerfile | 42 ++++ .../cd/builders/ticdc/centos7/Dockerfile | 50 ++++ .../cd/builders/ticdc/skaffold-centos7.yaml | 35 +++ dockerfiles/cd/builders/ticdc/skaffold.yaml | 36 +++ .../tidb-dashboard/skaffold-centos7.yaml | 72 ++++++ .../cd/builders/tidb-dashboard/skaffold.yaml | 55 +++++ .../cd/builders/tidb-operator/skaffold.yaml | 55 +++++ .../cd/builders/tidb/skaffold-centos7.yaml | 72 ++++++ dockerfiles/cd/builders/tidb/skaffold.yaml | 55 +++++ dockerfiles/cd/builders/tiflash/Dockerfile | 1 - .../cd/builders/tiflash/centos7/Dockerfile | 8 +- .../cd/builders/tiflash/skaffold-centos7.yaml | 24 ++ dockerfiles/cd/builders/tiflash/skaffold.yaml | 18 ++ dockerfiles/cd/builders/tiflow/Dockerfile | 1 - .../cd/builders/tiflow/centos7/Dockerfile | 3 + .../cd/builders/tiflow/skaffold-centos7.yaml | 71 ++++++ dockerfiles/cd/builders/tiflow/skaffold.yaml | 55 +++++ .../cd/builders/tikv/skaffold-centos7.yaml | 52 ++++ dockerfiles/cd/builders/tikv/skaffold.yaml | 49 ++++ 32 files changed, 1636 insertions(+), 533 deletions(-) create mode 100644 dockerfiles/cd/builders/ng-monitoring/skaffold-centos7.yaml create mode 100644 dockerfiles/cd/builders/ng-monitoring/skaffold.yaml create mode 100644 dockerfiles/cd/builders/pd/skaffold-centos7.yaml create mode 100644 dockerfiles/cd/builders/pd/skaffold.yaml delete mode 100644 dockerfiles/cd/builders/skaffold-centos7.yaml delete mode 100644 dockerfiles/cd/builders/skaffold.yaml create mode 100644 dockerfiles/cd/builders/tem/skaffold.yaml create mode 100644 dockerfiles/cd/builders/ticdc/Dockerfile create mode 100644 dockerfiles/cd/builders/ticdc/centos7/Dockerfile create mode 100644 dockerfiles/cd/builders/ticdc/skaffold-centos7.yaml create mode 100644 dockerfiles/cd/builders/ticdc/skaffold.yaml create mode 100644 dockerfiles/cd/builders/tidb-dashboard/skaffold-centos7.yaml create mode 100644 dockerfiles/cd/builders/tidb-dashboard/skaffold.yaml create mode 100644 dockerfiles/cd/builders/tidb-operator/skaffold.yaml create mode 100644 dockerfiles/cd/builders/tidb/skaffold-centos7.yaml create mode 100644 dockerfiles/cd/builders/tidb/skaffold.yaml create mode 100644 dockerfiles/cd/builders/tiflash/skaffold-centos7.yaml create mode 100644 dockerfiles/cd/builders/tiflash/skaffold.yaml create mode 100644 dockerfiles/cd/builders/tiflow/skaffold-centos7.yaml create mode 100644 dockerfiles/cd/builders/tiflow/skaffold.yaml create mode 100644 dockerfiles/cd/builders/tikv/skaffold-centos7.yaml create mode 100644 dockerfiles/cd/builders/tikv/skaffold.yaml diff --git a/.github/workflows/pull-cd-builder-images-centos7.yaml b/.github/workflows/pull-cd-builder-images-centos7.yaml index 495ddc1b..05739f79 100644 --- a/.github/workflows/pull-cd-builder-images-centos7.yaml +++ b/.github/workflows/pull-cd-builder-images-centos7.yaml @@ -6,16 +6,80 @@ on: paths: - "dockerfiles/cd/builders/*/centos7/Dockerfile" - "dockerfiles/cd/builders/*/centos7/*.Dockerfile" - - "dockerfiles/cd/builders/skaffold-centos7.yaml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + - "dockerfiles/cd/builders/*/skaffold-centos7.yaml" jobs: - skaffold-go: - name: build images with skaffold + define-matrix: + runs-on: ubuntu-latest + outputs: + changed_folders: ${{ steps.changed.outputs.folders }} + changed_go_builder_folders: ${{ steps.changed.outputs.go_builder_folders }} + changed_rust_builder_folders: ${{ steps.changed.outputs.rust_builder_folders }} + changed_cpp_builder_folders: ${{ steps.changed.outputs.cpp_builder_folders }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: "0" + - name: Parse changed scopes + id: changed + run: | + # Determine the base SHA depending on the event + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + elif [ "${{ github.event_name }}" = "push" ]; then + # For push events, compare with the previous commit + BASE_SHA=$(git rev-parse HEAD^) + HEAD_SHA=$(git rev-parse HEAD) + else + echo "Unsupported event: ${{ github.event_name }}" + exit 1 + fi + + echo "Base SHA: $BASE_SHA" + echo "Head SHA: $HEAD_SHA" + + # Get list of changed files with status + git diff --name-status $BASE_SHA $HEAD_SHA > changed_files.txt + echo "Changed Files with Status:" + cat changed_files.txt + + # Filter out deleted files and get only filenames and select the files in dockerfiles/cd/builders with `centos7` matched. + CHANGED_FILES=$(grep -v '^D' changed_files.txt | cut -f2- | grep -E '^dockerfiles/cd/builders/(.*Dockerfile|skaffold.*.yaml)' | grep centos7) + CHANGED_FOLDERS=$(echo "$CHANGED_FILES" | xargs -n1 dirname | sort --unique) + + # Convert the lists to JSON arrays + JSON_CHANGED_FILES=$(echo "$CHANGED_FILES" | jq -R -s -c 'split("\n") | map(select(length > 0))') + JSON_CHANGED_FOLDERS=$(echo "$CHANGED_FOLDERS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + # trim the prefix substring "dockerfiles/cd/builders/" from the JSON_CHANGED_FOLDERS + JSON_CHANGED_BUILDERS=$(echo "$JSON_CHANGED_FOLDERS" | jq -c 'map(sub("/centos7"; "")) | map(sub("dockerfiles/cd/builders/"; "")) | map(select(length > 0)) | unique') + + # golang type builders + JSON_ALL_GO_BUILDERS='["ng-monitoring", "pd","ticdc", "tidb", "tidb-dashboard", "tiflow"]' + JSON_ALL_RUST_BUILDERS='["tikv"]' + JSON_ALL_CPP_BUILDERS='["tiflash"]' + + ### Compute the building matrix on builder axies. + # Get intersection of arrays: JSON_ALL_GO_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_GO_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_GO_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + # Get intersection of arrays: JSON_ALL_RUST_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_RUST_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_RUST_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + # Get intersection of arrays: JSON_ALL_CPP_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_CPP_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_CPP_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + + # Set the output variable + echo "files=$JSON_CHANGED_FILES" >> $GITHUB_OUTPUT + echo "folders=$JSON_CHANGED_FOLDERS" >> $GITHUB_OUTPUT + echo "go_builder_folders=$JSON_CHANGED_FOLDERS_GO_TYPE" >> $GITHUB_OUTPUT + echo "rust_builder_folders=$JSON_CHANGED_FOLDERS_RUST_TYPE" >> $GITHUB_OUTPUT + echo "cpp_builder_folders=$JSON_CHANGED_FOLDERS_CPP_TYPE" >> $GITHUB_OUTPUT + + skaffold-golang: + name: build images for golang builders runs-on: ubuntu-latest + needs: define-matrix permissions: contents: read @@ -24,19 +88,28 @@ jobs: strategy: matrix: module: [builder-go] - go-profile: [go-1.23, go-1.22, go-1.21, go-1.20, go-1.19] + profile: [go-1.23, go-1.22, go-1.21, go-1.20, go-1.19] platform: [linux/amd64, linux/arm64] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_go_builder_folders) }} + exclude: + - builder: ticdc + profile: go-1.22 + - builder: ticdc + profile: go-1.21 + - builder: ticdc + profile: go-1.20 + - builder: ticdc + profile: go-1.19 steps: - name: Checkout sources uses: actions/checkout@v4 with: - fetch-depth: '0' - fetch-tags: 'true' + fetch-depth: "0" + fetch-tags: "true" - # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -54,42 +127,40 @@ jobs: sudo install skaffold /usr/local/bin/ - name: Build images - working-directory: dockerfiles/cd/builders + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} run: | skaffold build \ --filename=skaffold-centos7.yaml \ --build-concurrency=1 \ --cache-artifacts \ --default-repo=ghcr.io/pingcap-qe/cd/builders \ - --module=${{ matrix.module }} \ - --profile=${{ matrix.go-profile }} \ + --profile=${{ matrix.profile }} \ --platform=${{ matrix.platform }} \ --push=false - skaffold-nogo: - name: build images with skaffold + skaffold-rust: + name: build images for rust builders runs-on: ubuntu-latest - permissions: contents: read packages: write strategy: matrix: - # module: [builder-tiflash, builder-tikv] module: [builder-tikv] platform: [linux/amd64, linux/arm64] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_rust_builder_folders) }} + steps: - name: Checkout sources uses: actions/checkout@v4 with: - fetch-depth: '0' - fetch-tags: 'true' + fetch-depth: "0" + fetch-tags: "true" - # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -107,7 +178,7 @@ jobs: sudo install skaffold /usr/local/bin/ - name: Build images - working-directory: dockerfiles/cd/builders + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} run: | skaffold build \ --filename=skaffold-centos7.yaml \ @@ -117,3 +188,52 @@ jobs: --module=${{ matrix.module }} \ --platform=${{ matrix.platform }} \ --push=false + + skaffold-cpp: + name: skaffold build images for cpp builders + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + platform: [linux/amd64, linux/arm64] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_cpp_builder_folders) }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: "0" + fetch-tags: "true" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.13.2/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + + - name: Build images + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} + run: | + skaffold build \ + --filename=skaffold-centos7.yaml \ + --build-concurrency=1 \ + --cache-artifacts \ + --default-repo=ghcr.io/pingcap-qe/cd/builders \ + --platform=${{ matrix.platform }} \ + --push=false diff --git a/.github/workflows/pull-cd-builder-images.yaml b/.github/workflows/pull-cd-builder-images.yaml index d2f58830..14feae49 100644 --- a/.github/workflows/pull-cd-builder-images.yaml +++ b/.github/workflows/pull-cd-builder-images.yaml @@ -6,16 +6,80 @@ on: paths: - "dockerfiles/cd/builders/*/Dockerfile" - "dockerfiles/cd/builders/*/*.Dockerfile" - - "dockerfiles/cd/builders/skaffold.yaml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + - "dockerfiles/cd/builders/*/skaffold.yaml" jobs: - skaffold-go: - name: build images with skaffold + define-matrix: + runs-on: ubuntu-latest + outputs: + changed_folders: ${{ steps.changed.outputs.folders }} + changed_go_builder_folders: ${{ steps.changed.outputs.go_builder_folders }} + changed_rust_builder_folders: ${{ steps.changed.outputs.rust_builder_folders }} + changed_cpp_builder_folders: ${{ steps.changed.outputs.cpp_builder_folders }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: "0" + - name: Parse changed scopes + id: changed + run: | + # Determine the base SHA depending on the event + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + elif [ "${{ github.event_name }}" = "push" ]; then + # For push events, compare with the previous commit + BASE_SHA=$(git rev-parse HEAD^) + HEAD_SHA=$(git rev-parse HEAD) + else + echo "Unsupported event: ${{ github.event_name }}" + exit 1 + fi + + echo "Base SHA: $BASE_SHA" + echo "Head SHA: $HEAD_SHA" + + # Get list of changed files with status + git diff --name-status $BASE_SHA $HEAD_SHA > changed_files.txt + echo "Changed Files with Status:" + cat changed_files.txt + + # Filter out deleted files and get only filenames and select the files in dockerfiles/cd/builders without `centos7` matched. + CHANGED_FILES=$(grep -v '^D' changed_files.txt | cut -f2- | grep -E '^dockerfiles/cd/builders/(.*Dockerfile|skaffold.*.yaml)' | grep -v centos7) + CHANGED_FOLDERS=$(echo "$CHANGED_FILES" | xargs -n1 dirname | sort --unique) + + # Convert the lists to JSON arrays + JSON_CHANGED_FILES=$(echo "$CHANGED_FILES" | jq -R -s -c 'split("\n") | map(select(length > 0))') + JSON_CHANGED_FOLDERS=$(echo "$CHANGED_FOLDERS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + # trim the prefix substring "dockerfiles/cd/builders/" from the JSON_CHANGED_FOLDERS + JSON_CHANGED_BUILDERS=$(echo "$JSON_CHANGED_FOLDERS" | jq -c 'map(sub("dockerfiles/cd/builders/"; "")) | map(select(length > 0)) | unique') + + # golang type builders + JSON_ALL_GO_BUILDERS='["ng-monitoring", "pd", "tem", "ticdc", "tidb", "tidb-dashboard", "tidb-operator", "tiflow"]' + JSON_ALL_RUST_BUILDERS='["tikv"]' + JSON_ALL_CPP_BUILDERS='["tiflash"]' + + ### Compute the building matrix on builder axies. + # Get intersection of arrays: JSON_ALL_GO_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_GO_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_GO_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + # Get intersection of arrays: JSON_ALL_RUST_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_RUST_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_RUST_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + # Get intersection of arrays: JSON_ALL_CPP_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_CPP_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_CPP_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + + # Set the output variable + echo "files=$JSON_CHANGED_FILES" >> $GITHUB_OUTPUT + echo "folders=$JSON_CHANGED_FOLDERS" >> $GITHUB_OUTPUT + echo "go_builder_folders=$JSON_CHANGED_FOLDERS_GO_TYPE" >> $GITHUB_OUTPUT + echo "rust_builder_folders=$JSON_CHANGED_FOLDERS_RUST_TYPE" >> $GITHUB_OUTPUT + echo "cpp_builder_folders=$JSON_CHANGED_FOLDERS_CPP_TYPE" >> $GITHUB_OUTPUT + + skaffold-golang: + name: skaffold build images for golang builders runs-on: ubuntu-latest + needs: define-matrix permissions: contents: read @@ -24,19 +88,25 @@ jobs: strategy: matrix: module: [builder-go] - go-profile: [go-1.23, go-1.22, go-1.21] + profile: [go-1.23, go-1.22, go-1.21] platform: [linux/amd64, linux/arm64] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_go_builder_folders) }} + exclude: + - builder: ticdc + profile: go-1.22 + - builder: ticdc + profile: go-1.21 + steps: - name: Checkout sources uses: actions/checkout@v4 with: - fetch-depth: '0' - fetch-tags: 'true' + fetch-depth: "0" + fetch-tags: "true" - # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -54,40 +124,39 @@ jobs: sudo install skaffold /usr/local/bin/ - name: Build images - working-directory: dockerfiles/cd/builders + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} run: | skaffold build \ --build-concurrency=1 \ --cache-artifacts \ --default-repo=ghcr.io/pingcap-qe/cd/builders \ - --module=${{ matrix.module }} \ - --profile=${{ matrix.go-profile }} \ + --profile=${{ matrix.profile }} \ --platform=${{ matrix.platform }} \ --push=false - skaffold-nogo: - name: build images with skaffold + skaffold-rust: + name: build images for rust builders runs-on: ubuntu-latest - permissions: contents: read packages: write strategy: matrix: - module: [builder-tiflash, builder-tikv, builder-tikv-fips] + module: [builder-tikv, builder-tikv-fips] platform: [linux/amd64, linux/arm64] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_rust_builder_folders) }} + steps: - name: Checkout sources uses: actions/checkout@v4 with: - fetch-depth: '0' - fetch-tags: 'true' + fetch-depth: "0" + fetch-tags: "true" - # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -105,7 +174,7 @@ jobs: sudo install skaffold /usr/local/bin/ - name: Build images - working-directory: dockerfiles/cd/builders + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} run: | skaffold build \ --build-concurrency=1 \ @@ -114,3 +183,51 @@ jobs: --module=${{ matrix.module }} \ --platform=${{ matrix.platform }} \ --push=false + + skaffold-cpp: + name: skaffold build images for cpp builders + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + platform: [linux/amd64, linux/arm64] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_cpp_builder_folders) }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: "0" + fetch-tags: "true" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.13.2/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + + - name: Build images + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} + run: | + skaffold build \ + --build-concurrency=1 \ + --cache-artifacts \ + --default-repo=ghcr.io/pingcap-qe/cd/builders \ + --platform=${{ matrix.platform }} \ + --push=false diff --git a/.github/workflows/pull-cd-builder-images2.yaml b/.github/workflows/pull-cd-builder-images2.yaml index 6e2c749b..1c6613b5 100644 --- a/.github/workflows/pull-cd-builder-images2.yaml +++ b/.github/workflows/pull-cd-builder-images2.yaml @@ -55,7 +55,7 @@ jobs: if: contains(github.event.comment.body, 'msb-pd') run: just msb-pd - - name: Testing for CDC of tiflow + - name: Testing for new CDC if: contains(github.event.comment.body, 'msb-cdc') run: just msb-cdc diff --git a/.github/workflows/release-cd-builder-images-centos7.yaml b/.github/workflows/release-cd-builder-images-centos7.yaml index 0b8d6777..bd1c83a1 100644 --- a/.github/workflows/release-cd-builder-images-centos7.yaml +++ b/.github/workflows/release-cd-builder-images-centos7.yaml @@ -7,12 +7,80 @@ on: paths: - "dockerfiles/cd/builders/*/centos7/Dockerfile" - "dockerfiles/cd/builders/*/centos7/*.Dockerfile" - - "dockerfiles/cd/builders/skaffold-centos7.yaml" + - "dockerfiles/cd/builders/*/skaffold-centos7.yaml" jobs: - skaffold-go: - name: publish images with skaffold + define-matrix: runs-on: ubuntu-latest + outputs: + changed_folders: ${{ steps.changed.outputs.folders }} + changed_go_builder_folders: ${{ steps.changed.outputs.go_builder_folders }} + changed_rust_builder_folders: ${{ steps.changed.outputs.rust_builder_folders }} + changed_cpp_builder_folders: ${{ steps.changed.outputs.cpp_builder_folders }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: "0" + - name: Parse changed scopes + id: changed + run: | + # Determine the base SHA depending on the event + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + elif [ "${{ github.event_name }}" = "push" ]; then + # For push events, compare with the previous commit + BASE_SHA=$(git rev-parse HEAD^) + HEAD_SHA=$(git rev-parse HEAD) + else + echo "Unsupported event: ${{ github.event_name }}" + exit 1 + fi + + echo "Base SHA: $BASE_SHA" + echo "Head SHA: $HEAD_SHA" + + # Get list of changed files with status + git diff --name-status $BASE_SHA $HEAD_SHA > changed_files.txt + echo "Changed Files with Status:" + cat changed_files.txt + + # Filter out deleted files and get only filenames and select the files in dockerfiles/cd/builders with `centos7` matched. + CHANGED_FILES=$(grep -v '^D' changed_files.txt | cut -f2- | grep -E '^dockerfiles/cd/builders/(.*Dockerfile|skaffold.*.yaml)' | grep centos7) + CHANGED_FOLDERS=$(echo "$CHANGED_FILES" | xargs -n1 dirname | sort --unique) + + # Convert the lists to JSON arrays + JSON_CHANGED_FILES=$(echo "$CHANGED_FILES" | jq -R -s -c 'split("\n") | map(select(length > 0))') + JSON_CHANGED_FOLDERS=$(echo "$CHANGED_FOLDERS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + # trim the prefix substring "dockerfiles/cd/builders/" from the JSON_CHANGED_FOLDERS + JSON_CHANGED_BUILDERS=$(echo "$JSON_CHANGED_FOLDERS" | jq -c 'map(sub("/centos7"; "")) | map(sub("dockerfiles/cd/builders/"; "")) | map(select(length > 0)) | unique') + + # golang type builders + JSON_ALL_GO_BUILDERS='["ng-monitoring", "pd","ticdc", "tidb", "tidb-dashboard", "tiflow"]' + JSON_ALL_RUST_BUILDERS='["tikv"]' + JSON_ALL_CPP_BUILDERS='["tiflash"]' + + ### Compute the building matrix on builder axies. + # Get intersection of arrays: JSON_ALL_GO_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_GO_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_GO_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + # Get intersection of arrays: JSON_ALL_RUST_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_RUST_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_RUST_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + # Get intersection of arrays: JSON_ALL_CPP_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_CPP_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_CPP_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + + # Set the output variable + echo "files=$JSON_CHANGED_FILES" >> $GITHUB_OUTPUT + echo "folders=$JSON_CHANGED_FOLDERS" >> $GITHUB_OUTPUT + echo "go_builder_folders=$JSON_CHANGED_FOLDERS_GO_TYPE" >> $GITHUB_OUTPUT + echo "rust_builder_folders=$JSON_CHANGED_FOLDERS_RUST_TYPE" >> $GITHUB_OUTPUT + echo "cpp_builder_folders=$JSON_CHANGED_FOLDERS_CPP_TYPE" >> $GITHUB_OUTPUT + + skaffold-golang: + name: publish images for golang builders + runs-on: ubuntu-latest + needs: define-matrix permissions: contents: read @@ -21,18 +89,28 @@ jobs: strategy: matrix: module: [builder-go] - go-profile: [go-1.23, go-1.22, go-1.21, go-1.20, go-1.19] + profile: [go-1.23, go-1.22, go-1.21, go-1.20, go-1.19] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_go_builder_folders) }} + exclude: + - builder: ticdc + profile: go-1.22 + - builder: ticdc + profile: go-1.21 + - builder: ticdc + profile: go-1.20 + - builder: ticdc + profile: go-1.19 + steps: - name: Checkout sources uses: actions/checkout@v4 with: - fetch-depth: '0' - fetch-tags: 'true' + fetch-depth: "0" + fetch-tags: "true" - # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -50,36 +128,36 @@ jobs: sudo install skaffold /usr/local/bin/ - name: Publish images - working-directory: dockerfiles/cd/builders + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} run: | skaffold build \ --filename=skaffold-centos7.yaml \ --build-concurrency=1 \ --cache-artifacts \ --default-repo=ghcr.io/pingcap-qe/cd/builders \ - --module=${{ matrix.module }} \ - --profile=${{ matrix.go-profile }} - skaffold-nogo: - name: publish images with skaffold - runs-on: ubuntu-latest + --profile=${{ matrix.profile }} + skaffold-rust: + name: publish images for rust builders + runs-on: ubuntu-latest permissions: contents: read packages: write strategy: matrix: - module: [builder-tiflash, builder-tikv] + module: [builder-tikv] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_rust_builder_folders) }} + steps: - name: Checkout sources uses: actions/checkout@v4 with: - fetch-depth: '0' - fetch-tags: 'true' + fetch-depth: "0" + fetch-tags: "true" - # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -97,7 +175,7 @@ jobs: sudo install skaffold /usr/local/bin/ - name: Publish images - working-directory: dockerfiles/cd/builders + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} run: | skaffold build \ --filename=skaffold-centos7.yaml \ @@ -105,3 +183,49 @@ jobs: --cache-artifacts \ --default-repo=ghcr.io/pingcap-qe/cd/builders \ --module=${{ matrix.module }} + + skaffold-cpp: + name: skaffold build images for cpp builders + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_cpp_builder_folders) }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: "0" + fetch-tags: "true" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.13.2/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + + - name: Build images + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} + run: | + skaffold build \ + --filename=skaffold-centos7.yaml \ + --build-concurrency=1 \ + --cache-artifacts \ + --default-repo=ghcr.io/pingcap-qe/cd/builders diff --git a/.github/workflows/release-cd-builder-images.yaml b/.github/workflows/release-cd-builder-images.yaml index 1a362574..32427851 100644 --- a/.github/workflows/release-cd-builder-images.yaml +++ b/.github/workflows/release-cd-builder-images.yaml @@ -7,12 +7,80 @@ on: paths: - "dockerfiles/cd/builders/*/Dockerfile" - "dockerfiles/cd/builders/*/*.Dockerfile" - - "dockerfiles/cd/builders/skaffold.yaml" + - "dockerfiles/cd/builders/*/skaffold.yaml" jobs: - skaffold-go: - name: publish images with skaffold + define-matrix: runs-on: ubuntu-latest + outputs: + changed_folders: ${{ steps.changed.outputs.folders }} + changed_go_builder_folders: ${{ steps.changed.outputs.go_builder_folders }} + changed_rust_builder_folders: ${{ steps.changed.outputs.rust_builder_folders }} + changed_cpp_builder_folders: ${{ steps.changed.outputs.cpp_builder_folders }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: "0" + - name: Parse changed scopes + id: changed + run: | + # Determine the base SHA depending on the event + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + elif [ "${{ github.event_name }}" = "push" ]; then + # For push events, compare with the previous commit + BASE_SHA=$(git rev-parse HEAD^) + HEAD_SHA=$(git rev-parse HEAD) + else + echo "Unsupported event: ${{ github.event_name }}" + exit 1 + fi + + echo "Base SHA: $BASE_SHA" + echo "Head SHA: $HEAD_SHA" + + # Get list of changed files with status + git diff --name-status $BASE_SHA $HEAD_SHA > changed_files.txt + echo "Changed Files with Status:" + cat changed_files.txt + + # Filter out deleted files and get only filenames and select the files in dockerfiles/cd/builders without `centos7` matched. + CHANGED_FILES=$(grep -v '^D' changed_files.txt | cut -f2- | grep -E '^dockerfiles/cd/builders/(.*Dockerfile|skaffold.*.yaml)' | grep -v centos7) + CHANGED_FOLDERS=$(echo "$CHANGED_FILES" | xargs -n1 dirname | sort --unique) + + # Convert the lists to JSON arrays + JSON_CHANGED_FILES=$(echo "$CHANGED_FILES" | jq -R -s -c 'split("\n") | map(select(length > 0))') + JSON_CHANGED_FOLDERS=$(echo "$CHANGED_FOLDERS" | jq -R -s -c 'split("\n") | map(select(length > 0))') + # trim the prefix substring "dockerfiles/cd/builders/" from the JSON_CHANGED_FOLDERS + JSON_CHANGED_BUILDERS=$(echo "$JSON_CHANGED_FOLDERS" | jq -c 'map(sub("dockerfiles/cd/builders/"; "")) | map(select(length > 0)) | unique') + + # golang type builders + JSON_ALL_GO_BUILDERS='["ng-monitoring", "pd", "tem", "ticdc", "tidb", "tidb-dashboard", "tidb-operator", "tiflow"]' + JSON_ALL_RUST_BUILDERS='["tikv"]' + JSON_ALL_CPP_BUILDERS='["tiflash"]' + + ### Compute the building matrix on builder axies. + # Get intersection of arrays: JSON_ALL_GO_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_GO_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_GO_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + # Get intersection of arrays: JSON_ALL_RUST_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_RUST_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_RUST_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + # Get intersection of arrays: JSON_ALL_CPP_BUILDERS, JSON_CHANGED_BUILDERS + JSON_CHANGED_FOLDERS_CPP_TYPE=$(jq --null-input --argjson changed "$JSON_CHANGED_BUILDERS" --argjson ALL "$JSON_ALL_CPP_BUILDERS" -R -s -c '$changed - ($changed - $ALL)' ) + + # Set the output variable + echo "files=$JSON_CHANGED_FILES" >> $GITHUB_OUTPUT + echo "folders=$JSON_CHANGED_FOLDERS" >> $GITHUB_OUTPUT + echo "go_builder_folders=$JSON_CHANGED_FOLDERS_GO_TYPE" >> $GITHUB_OUTPUT + echo "rust_builder_folders=$JSON_CHANGED_FOLDERS_RUST_TYPE" >> $GITHUB_OUTPUT + echo "cpp_builder_folders=$JSON_CHANGED_FOLDERS_CPP_TYPE" >> $GITHUB_OUTPUT + + skaffold-golang: + name: skaffold publish images for golang builders + runs-on: ubuntu-latest + needs: define-matrix permissions: contents: read @@ -21,18 +89,24 @@ jobs: strategy: matrix: module: [builder-go] - go-profile: [go-1.23, go-1.22, go-1.21] + profile: [go-1.23, go-1.22, go-1.21] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_go_builder_folders) }} + exclude: + - builder: ticdc + profile: go-1.22 + - builder: ticdc + profile: go-1.21 + steps: - name: Checkout sources uses: actions/checkout@v4 with: - fetch-depth: '0' - fetch-tags: 'true' + fetch-depth: "0" + fetch-tags: "true" - # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -50,35 +124,36 @@ jobs: sudo install skaffold /usr/local/bin/ - name: Publish images - working-directory: dockerfiles/cd/builders + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} run: | skaffold build \ - --build-concurrency 1 \ + --build-concurrency=1 \ --cache-artifacts \ - --default-repo ghcr.io/pingcap-qe/cd/builders \ - --module ${{ matrix.module }} \ - --profile ${{ matrix.go-profile }} - skaffold-nogo: - name: publish images with skaffold - runs-on: ubuntu-latest + --default-repo=ghcr.io/pingcap-qe/cd/builders \ + --profile=${{ matrix.profile }} + skaffold-rust: + name: publish images for rust builders + runs-on: ubuntu-latest permissions: contents: read packages: write + strategy: matrix: - module: [builder-tiflash, builder-tikv, builder-tikv-fips] + module: [builder-tikv, builder-tikv-fips] + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_rust_builder_folders) }} + steps: - name: Checkout sources uses: actions/checkout@v4 with: - fetch-depth: '0' - fetch-tags: 'true' + fetch-depth: "0" + fetch-tags: "true" - # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 @@ -96,10 +171,55 @@ jobs: sudo install skaffold /usr/local/bin/ - name: Publish images - working-directory: dockerfiles/cd/builders + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} + run: | + skaffold build \ + --build-concurrency=1 \ + --cache-artifacts \ + --default-repo=ghcr.io/pingcap-qe/cd/builders \ + --module=${{ matrix.module }} + + skaffold-cpp: + name: publish images for cpp builders + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + builder: ${{ fromJSON(needs.define-matrix.outputs.changed_cpp_builder_folders) }} + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + with: + fetch-depth: "0" + fetch-tags: "true" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.13.2/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + + - name: Build images + working-directory: dockerfiles/cd/builders/${{ matrix.builder }} run: | skaffold build \ - --build-concurrency 1 \ + --build-concurrency=1 \ --cache-artifacts \ - --default-repo ghcr.io/pingcap-qe/cd/builders \ - --module ${{ matrix.module }} + --default-repo=ghcr.io/pingcap-qe/cd/builders diff --git a/Justfile b/Justfile index 8df04344..1a499cde 100644 --- a/Justfile +++ b/Justfile @@ -7,9 +7,12 @@ msb-tiflash: (_clone "tiflash" "https://github.com/pingcap/tiflash" "master") msb-dm: (_clone "tiflow" "https://github.com/pingcap/tiflow.git" "master") docker build --load -t localhost/dm:local-build -f dockerfiles/cd/builders/tiflow/Dockerfile --target final-dm ../tiflow -msb-cdc: (_clone "tiflow" "https://github.com/pingcap/tiflow.git" "master") +msb-cdc-old: (_clone "tiflow" "https://github.com/pingcap/tiflow.git" "master") docker build --load -t localhost/cdc:local-build -f dockerfiles/cd/builders/tiflow/Dockerfile --target final-cdc ../tiflow +msb-cdc: (_clone "ticdc" "https://github.com/pingcap/ticdc.git" "master") + docker build --load -t localhost/cdc:local-build -f dockerfiles/cd/builders/ticdc/Dockerfile ../ticdc + msb-tidb: (_clone_without_submodules "tidb" "https://github.com/pingcap/tidb.git" "master") docker build --load -t localhost/tidb:local-build -f dockerfiles/cd/builders/tidb/Dockerfile ../tidb diff --git a/dockerfiles/cd/builders/ng-monitoring/skaffold-centos7.yaml b/dockerfiles/cd/builders/ng-monitoring/skaffold-centos7.yaml new file mode 100644 index 00000000..5da49ec4 --- /dev/null +++ b/dockerfiles/cd/builders/ng-monitoring/skaffold-centos7.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: ng-monitoring + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: centos7/Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}-centos7" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 + - name: go-1.20 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.20" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.20.14 + - name: go-1.19 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.19" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.19.13 diff --git a/dockerfiles/cd/builders/ng-monitoring/skaffold.yaml b/dockerfiles/cd/builders/ng-monitoring/skaffold.yaml new file mode 100644 index 00000000..939bcd81 --- /dev/null +++ b/dockerfiles/cd/builders/ng-monitoring/skaffold.yaml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: ng-monitoring + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 diff --git a/dockerfiles/cd/builders/pd/skaffold-centos7.yaml b/dockerfiles/cd/builders/pd/skaffold-centos7.yaml new file mode 100644 index 00000000..000b5089 --- /dev/null +++ b/dockerfiles/cd/builders/pd/skaffold-centos7.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: pd + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: centos7/Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}-centos7" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 + - name: go-1.20 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.20" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.20.14 + - name: go-1.19 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.19" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.19.13 diff --git a/dockerfiles/cd/builders/pd/skaffold.yaml b/dockerfiles/cd/builders/pd/skaffold.yaml new file mode 100644 index 00000000..bd9bfb65 --- /dev/null +++ b/dockerfiles/cd/builders/pd/skaffold.yaml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: pd + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 diff --git a/dockerfiles/cd/builders/skaffold-centos7.yaml b/dockerfiles/cd/builders/skaffold-centos7.yaml deleted file mode 100644 index 708ad6df..00000000 --- a/dockerfiles/cd/builders/skaffold-centos7.yaml +++ /dev/null @@ -1,228 +0,0 @@ ---- -apiVersion: skaffold/v4beta6 -kind: Config -metadata: - name: builder-go -build: - artifacts: - - image: tidb - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tidb/centos7/Dockerfile - target: builder - - image: tiflow - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tiflow/centos7/Dockerfile - target: builder - - image: pd - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: pd/centos7/Dockerfile - target: builder - - image: ng-monitoring - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: ng-monitoring/centos7/Dockerfile - target: builder - - image: tidb-dashboard - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tidb-dashboard/centos7/Dockerfile - target: builder - tagPolicy: - customTemplate: - template: "{{ .SHA }}-centos7" - components: - - name: SHA - gitCommit: - variant: Tags - local: - useDockerCLI: true - useBuildkit: true - concurrency: 0 - tryImportMissing: true -profiles: - - name: go-1.23 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-centos7-go1.23" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: &value-go1-23 - # renovate: datasource=docker depName=golang - GOLANG_VERSION: 1.23.4 - - op: add - path: /build/artifacts/1/docker/buildArgs - value: *value-go1-23 - - op: add - path: /build/artifacts/2/docker/buildArgs - value: *value-go1-23 - - op: add - path: /build/artifacts/3/docker/buildArgs - value: *value-go1-23 - - op: add - path: /build/artifacts/4/docker/buildArgs - value: *value-go1-23 - - name: go-1.22 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-centos7-go1.22" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: &value-go1-22 - # renovate: datasource=docker depName=golang - GOLANG_VERSION: 1.22.10 - - op: add - path: /build/artifacts/1/docker/buildArgs - value: *value-go1-22 - - op: add - path: /build/artifacts/2/docker/buildArgs - value: *value-go1-22 - - op: add - path: /build/artifacts/3/docker/buildArgs - value: *value-go1-22 - - op: add - path: /build/artifacts/4/docker/buildArgs - value: *value-go1-22 - - name: go-1.21 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-centos7-go1.21" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: &value-go1-21 - GOLANG_VERSION: 1.21.13 - - op: add - path: /build/artifacts/1/docker/buildArgs - value: *value-go1-21 - - op: add - path: /build/artifacts/2/docker/buildArgs - value: *value-go1-21 - - op: add - path: /build/artifacts/3/docker/buildArgs - value: *value-go1-21 - - op: add - path: /build/artifacts/4/docker/buildArgs - value: *value-go1-21 - - name: go-1.20 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-centos7-go1.20" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: &value-go1-20 - GOLANG_VERSION: 1.20.14 - - op: add - path: /build/artifacts/1/docker/buildArgs - value: *value-go1-20 - - op: add - path: /build/artifacts/2/docker/buildArgs - value: *value-go1-20 - - op: add - path: /build/artifacts/3/docker/buildArgs - value: *value-go1-20 - - op: add - path: /build/artifacts/4/docker/buildArgs - value: *value-go1-20 - - name: go-1.19 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-centos7-go1.19" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: &value-go1-19 - GOLANG_VERSION: 1.19.13 - - op: add - path: /build/artifacts/1/docker/buildArgs - value: *value-go1-19 - - op: add - path: /build/artifacts/2/docker/buildArgs - value: *value-go1-19 - - op: add - path: /build/artifacts/3/docker/buildArgs - value: *value-go1-19 - - op: add - path: /build/artifacts/4/docker/buildArgs - value: *value-go1-19 ---- -apiVersion: skaffold/v4beta6 -kind: Config -metadata: - name: builder-tikv -build: - artifacts: - - image: tikv - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tikv/centos7/Dockerfile - target: builder - local: - useDockerCLI: true - useBuildkit: true - concurrency: 0 - tryImportMissing: true - tagPolicy: - customTemplate: - template: "{{ .SHA }}-centos7" - components: - - name: SHA - gitCommit: - variant: Tags -profiles: - - name: devtoolset8 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-centos7-devtoolset8" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: - DEVTOOLSET_VER: 8 - - name: devtoolset9 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-centos7-devtoolset9" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: - DEVTOOLSET_VER: 9 - - name: devtoolset10 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-centos7-devtoolset10" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: - DEVTOOLSET_VER: 10 ---- -apiVersion: skaffold/v4beta6 -kind: Config -metadata: - name: builder-tiflash -build: - artifacts: - - image: tiflash - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tiflash/centos7/Dockerfile - target: builder - local: - useDockerCLI: true - useBuildkit: true - concurrency: 0 - tryImportMissing: true - tagPolicy: - customTemplate: - template: "{{ .SHA }}-centos7" - components: - - name: SHA - gitCommit: - variant: Tags diff --git a/dockerfiles/cd/builders/skaffold.yaml b/dockerfiles/cd/builders/skaffold.yaml deleted file mode 100644 index d33f3c1b..00000000 --- a/dockerfiles/cd/builders/skaffold.yaml +++ /dev/null @@ -1,204 +0,0 @@ -# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json ---- -apiVersion: skaffold/v4beta6 -kind: Config -metadata: - name: builder-go -build: - artifacts: - - image: tidb - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tidb/Dockerfile - target: builder - - image: tiflow - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tiflow/Dockerfile - target: builder - - image: pd - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: pd/Dockerfile - target: builder - - image: ng-monitoring - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: ng-monitoring/Dockerfile - target: builder - - image: tidb-dashboard - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tidb-dashboard/Dockerfile - target: builder - - image: tidb-operator - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tidb-operator/Dockerfile - target: builder - - image: tem - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tem/Dockerfile - target: builder - tagPolicy: - customTemplate: - template: "{{ .SHA }}" - components: - - name: SHA - gitCommit: - variant: Tags - local: - useDockerCLI: true - useBuildkit: true - concurrency: 0 - tryImportMissing: true -profiles: - - name: go-1.23 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-go1.23" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: &value-go1-23 - # renovate: datasource=docker depName=golang - GOLANG_VERSION: 1.23.4 - - op: add - path: /build/artifacts/1/docker/buildArgs - value: *value-go1-23 - - op: add - path: /build/artifacts/2/docker/buildArgs - value: *value-go1-23 - - op: add - path: /build/artifacts/3/docker/buildArgs - value: *value-go1-23 - - op: add - path: /build/artifacts/4/docker/buildArgs - value: *value-go1-23 - - op: add - path: /build/artifacts/5/docker/buildArgs - value: *value-go1-23 - - op: add - path: /build/artifacts/6/docker/buildArgs - value: *value-go1-23 - - name: go-1.22 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-go1.22" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: &value-go1-22 - # renovate: datasource=docker depName=golang - GOLANG_VERSION: 1.22.10 - - op: add - path: /build/artifacts/1/docker/buildArgs - value: *value-go1-22 - - op: add - path: /build/artifacts/2/docker/buildArgs - value: *value-go1-22 - - op: add - path: /build/artifacts/3/docker/buildArgs - value: *value-go1-22 - - op: add - path: /build/artifacts/4/docker/buildArgs - value: *value-go1-22 - - op: add - path: /build/artifacts/5/docker/buildArgs - value: *value-go1-22 - - op: add - path: /build/artifacts/6/docker/buildArgs - value: *value-go1-22 - - name: go-1.21 - patches: - - op: replace - path: /build/tagPolicy/customTemplate/template - value: "{{ .SHA }}-go1.21" - - op: add - path: /build/artifacts/0/docker/buildArgs - value: &value-go1-21 - GOLANG_VERSION: 1.21.13 - - op: add - path: /build/artifacts/1/docker/buildArgs - value: *value-go1-21 - - op: add - path: /build/artifacts/2/docker/buildArgs - value: *value-go1-21 - - op: add - path: /build/artifacts/3/docker/buildArgs - value: *value-go1-21 - - op: add - path: /build/artifacts/4/docker/buildArgs - value: *value-go1-21 - - op: add - path: /build/artifacts/5/docker/buildArgs - value: *value-go1-21 - - op: add - path: /build/artifacts/6/docker/buildArgs - value: *value-go1-21 ---- -apiVersion: skaffold/v4beta6 -kind: Config -metadata: - name: builder-tikv -build: - artifacts: - - image: tikv - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tikv/Dockerfile - target: builder - local: - useDockerCLI: true - useBuildkit: true - concurrency: 0 - tryImportMissing: true - tagPolicy: - customTemplate: - template: "{{ .SHA }}" - components: - - name: SHA - gitCommit: - variant: Tags ---- -apiVersion: skaffold/v4beta6 -kind: Config -metadata: - name: builder-tiflash -build: - artifacts: - - image: tiflash - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tiflash/Dockerfile - target: builder - local: - useDockerCLI: true - useBuildkit: true - concurrency: 0 - tryImportMissing: true ---- -apiVersion: skaffold/v4beta6 -kind: Config -metadata: - name: builder-tikv-fips -build: - artifacts: - - image: tikv - platforms: [linux/amd64, linux/arm64] - docker: - dockerfile: tikv/fips.Dockerfile - target: builder - tagPolicy: - customTemplate: - template: "{{ .SHA }}-fips" - components: - - name: SHA - gitCommit: - variant: Tags - local: - useDockerCLI: true - useBuildkit: true - concurrency: 0 - tryImportMissing: true diff --git a/dockerfiles/cd/builders/tem/skaffold.yaml b/dockerfiles/cd/builders/tem/skaffold.yaml new file mode 100644 index 00000000..ec25e988 --- /dev/null +++ b/dockerfiles/cd/builders/tem/skaffold.yaml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tem + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 diff --git a/dockerfiles/cd/builders/ticdc/Dockerfile b/dockerfiles/cd/builders/ticdc/Dockerfile new file mode 100644 index 00000000..198fb9e2 --- /dev/null +++ b/dockerfiles/cd/builders/ticdc/Dockerfile @@ -0,0 +1,42 @@ +# build requires: +# - docker >= v20.10 +# +# build steps: +# - git clone --recurse-submodules https://github.com/pingcap/ticdc.git ticdc +# - docker build -t ticdc -f Dockerfile ./ticdc + +########### stage: builder +FROM quay.io/rockylinux/rockylinux:8.10.20240528 as builder +LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" +LABEL org.opencontainers.image.description "binary builder for TiFlow" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" + +# install packages. +RUN --mount=type=cache,target=/var/cache/dnf \ + dnf upgrade-minimal -y && \ + dnf install -y make git findutils + +# install golang toolchain +# renovate: datasource=docker depName=golang +ARG GOLANG_VERSION=1.23.4 +RUN OS=linux; ARCH=$([ "$(arch)" = "x86_64" ] && echo amd64 || echo arm64); \ + curl -fsSL https://dl.google.com/go/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz | tar -C /usr/local -xz +ENV PATH /usr/local/go/bin/:$PATH +LABEL go-version="${GOLANG_VERSION}" + +########### stage: building-cdc +FROM builder as building +COPY . /ws +ARG GOPROXY +RUN GOPROXY=${GOPROXY} make cdc -C /ws +RUN /ws/bin/cdc version + +########### stage: Final image +FROM ghcr.io/pingcap-qe/bases/tools-base:v1.9.2 +COPY --from=building /ws/bin/cdc /cdc + +EXPOSE 8300 +CMD ["/cdc"] + +# test it +RUN /cdc version diff --git a/dockerfiles/cd/builders/ticdc/centos7/Dockerfile b/dockerfiles/cd/builders/ticdc/centos7/Dockerfile new file mode 100644 index 00000000..e944729a --- /dev/null +++ b/dockerfiles/cd/builders/ticdc/centos7/Dockerfile @@ -0,0 +1,50 @@ +# build requires: +# - docker >= v20.10 +# +# build steps: +# - git clone --recurse-submodules https://github.com/pingcap/ticdc.git ticdc +# - docker build -t ticdc -f Dockerfile ./ticdc + +########### stage: Builder +FROM centos:7.9.2009 as builder +LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" +LABEL org.opencontainers.image.description "binary builder for TiFlow" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" + +# CentOS 7 has reached EOL +RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \ + && sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo \ + && sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo + +# install packages. +RUN yum update --nogpgcheck -y && \ + yum install --nogpgcheck -y epel-release deltarpm && \ + yum update --nogpgcheck -y && \ + yum groupinstall --nogpgcheck -y "Development Tools" && \ + yum clean all + +# install golang toolchain +# renovate: datasource=docker depName=golang +ARG GOLANG_VERSION=1.23.4 +RUN OS=linux; ARCH=$([ "$(arch)" = "x86_64" ] && echo amd64 || echo arm64); \ + curl -fsSL https://dl.google.com/go/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz | tar -C /usr/local -xz +ENV PATH /usr/local/go/bin/:$PATH +LABEL go-version="${GOLANG_VERSION}" + +########### stage: Buiding +FROM builder as building + +COPY . /ws +ARG GOPROXY +RUN GOPROXY=${GOPROXY} make cdc -C /ws +RUN /ws/bin/cdc version + +########### stage: Final image +FROM ghcr.io/pingcap-qe/bases/tools-base:v1.9.2 + +COPY --from=building /ws/bin/cdc /cdc +EXPOSE 8300 +CMD ["/cdc"] + +# test it +RUN /cdc version diff --git a/dockerfiles/cd/builders/ticdc/skaffold-centos7.yaml b/dockerfiles/cd/builders/ticdc/skaffold-centos7.yaml new file mode 100644 index 00000000..54ff194d --- /dev/null +++ b/dockerfiles/cd/builders/ticdc/skaffold-centos7.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: ticdc + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}-centos7" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 diff --git a/dockerfiles/cd/builders/ticdc/skaffold.yaml b/dockerfiles/cd/builders/ticdc/skaffold.yaml new file mode 100644 index 00000000..5e3a4bc4 --- /dev/null +++ b/dockerfiles/cd/builders/ticdc/skaffold.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: ticdc + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 diff --git a/dockerfiles/cd/builders/tidb-dashboard/skaffold-centos7.yaml b/dockerfiles/cd/builders/tidb-dashboard/skaffold-centos7.yaml new file mode 100644 index 00000000..d584ba12 --- /dev/null +++ b/dockerfiles/cd/builders/tidb-dashboard/skaffold-centos7.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tidb-dashboard + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: centos7/Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}-centos7" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 + - name: go-1.20 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.20" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.20.14 + - name: go-1.19 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.19" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.19.13 diff --git a/dockerfiles/cd/builders/tidb-dashboard/skaffold.yaml b/dockerfiles/cd/builders/tidb-dashboard/skaffold.yaml new file mode 100644 index 00000000..91ab98ea --- /dev/null +++ b/dockerfiles/cd/builders/tidb-dashboard/skaffold.yaml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tidb-dashboard + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 diff --git a/dockerfiles/cd/builders/tidb-operator/skaffold.yaml b/dockerfiles/cd/builders/tidb-operator/skaffold.yaml new file mode 100644 index 00000000..5a9b69f6 --- /dev/null +++ b/dockerfiles/cd/builders/tidb-operator/skaffold.yaml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tidb-operator + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 diff --git a/dockerfiles/cd/builders/tidb/skaffold-centos7.yaml b/dockerfiles/cd/builders/tidb/skaffold-centos7.yaml new file mode 100644 index 00000000..90e85c5f --- /dev/null +++ b/dockerfiles/cd/builders/tidb/skaffold-centos7.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tidb + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: centos7/Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}-centos7" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 + - name: go-1.20 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.20" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.20.14 + - name: go-1.19 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.19" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.19.13 diff --git a/dockerfiles/cd/builders/tidb/skaffold.yaml b/dockerfiles/cd/builders/tidb/skaffold.yaml new file mode 100644 index 00000000..2a16f476 --- /dev/null +++ b/dockerfiles/cd/builders/tidb/skaffold.yaml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tidb + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 diff --git a/dockerfiles/cd/builders/tiflash/Dockerfile b/dockerfiles/cd/builders/tiflash/Dockerfile index 8b2207ef..02c3b311 100644 --- a/dockerfiles/cd/builders/tiflash/Dockerfile +++ b/dockerfiles/cd/builders/tiflash/Dockerfile @@ -11,7 +11,6 @@ LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" LABEL org.opencontainers.image.description "binary builder for TiFlash" LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" - # FIXME: replace with official repo and tag. # renovate: datasource=github-tags depName=pingcap/tiflash ARG TIFLASH_REPO=pingcap/tiflash diff --git a/dockerfiles/cd/builders/tiflash/centos7/Dockerfile b/dockerfiles/cd/builders/tiflash/centos7/Dockerfile index 45c52d0f..387281c8 100644 --- a/dockerfiles/cd/builders/tiflash/centos7/Dockerfile +++ b/dockerfiles/cd/builders/tiflash/centos7/Dockerfile @@ -7,15 +7,15 @@ ########### stage: builder FROM centos:7.9.2009 as builder +LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" +LABEL org.opencontainers.image.description "binary builder for TiFlash" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" + # CentOS 7 has reached EOL RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \ && sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo \ && sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo -LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" -LABEL org.opencontainers.image.description "binary builder for TiFlash" -LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" - ARG TIFLASH_VER=v8.3.0 RUN FILE=$([ "$(arch)" = "aarch64" ] && echo "bake_llvm_base_aarch64.sh" || echo "bake_llvm_base_amd64.sh"); \ TAR_DIR="tiflash-$(echo $TIFLASH_VER | sed 's/v//')"; \ diff --git a/dockerfiles/cd/builders/tiflash/skaffold-centos7.yaml b/dockerfiles/cd/builders/tiflash/skaffold-centos7.yaml new file mode 100644 index 00000000..c6772336 --- /dev/null +++ b/dockerfiles/cd/builders/tiflash/skaffold-centos7.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tiflash + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: centos7/Dockerfile + target: builder + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true + tagPolicy: + customTemplate: + template: "{{ .SHA }}-centos7" + components: + - name: SHA + gitCommit: + variant: Tags diff --git a/dockerfiles/cd/builders/tiflash/skaffold.yaml b/dockerfiles/cd/builders/tiflash/skaffold.yaml new file mode 100644 index 00000000..91158e29 --- /dev/null +++ b/dockerfiles/cd/builders/tiflash/skaffold.yaml @@ -0,0 +1,18 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tiflash + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true diff --git a/dockerfiles/cd/builders/tiflow/Dockerfile b/dockerfiles/cd/builders/tiflow/Dockerfile index 684cb4b3..0aab0ef7 100644 --- a/dockerfiles/cd/builders/tiflow/Dockerfile +++ b/dockerfiles/cd/builders/tiflow/Dockerfile @@ -7,7 +7,6 @@ ########### stage: builder-cdc FROM quay.io/rockylinux/rockylinux:8.10.20240528 as builder-cdc - # install packages. RUN --mount=type=cache,target=/var/cache/dnf \ dnf upgrade-minimal -y && \ diff --git a/dockerfiles/cd/builders/tiflow/centos7/Dockerfile b/dockerfiles/cd/builders/tiflow/centos7/Dockerfile index 9fc769d2..c837bf4e 100644 --- a/dockerfiles/cd/builders/tiflow/centos7/Dockerfile +++ b/dockerfiles/cd/builders/tiflow/centos7/Dockerfile @@ -47,6 +47,9 @@ RUN node --version && npm --version && npm install -g yarn ########### stage: Builder - total FROM builder-dm as builder +LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" +LABEL org.opencontainers.image.description "binary builder for TiFlow" +LABEL org.opencontainers.image.source "https://github.com/PingCAP-QE/artifacts" ########### stage: Buiding cdc FROM builder-cdc as building-cdc diff --git a/dockerfiles/cd/builders/tiflow/skaffold-centos7.yaml b/dockerfiles/cd/builders/tiflow/skaffold-centos7.yaml new file mode 100644 index 00000000..d7691b73 --- /dev/null +++ b/dockerfiles/cd/builders/tiflow/skaffold-centos7.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tiflow + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: centos7/Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}-centos7" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 + - name: go-1.20 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.20" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.20.14 + - name: go-1.19 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-go1.19" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.19.13 diff --git a/dockerfiles/cd/builders/tiflow/skaffold.yaml b/dockerfiles/cd/builders/tiflow/skaffold.yaml new file mode 100644 index 00000000..e63b3a9a --- /dev/null +++ b/dockerfiles/cd/builders/tiflow/skaffold.yaml @@ -0,0 +1,55 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tiflow + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +profiles: + - name: go-1.23 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.23" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.23.4 + - name: go-1.22 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.22" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.22.10 + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-go1.21" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + GOLANG_VERSION: 1.21.13 diff --git a/dockerfiles/cd/builders/tikv/skaffold-centos7.yaml b/dockerfiles/cd/builders/tikv/skaffold-centos7.yaml new file mode 100644 index 00000000..00027408 --- /dev/null +++ b/dockerfiles/cd/builders/tikv/skaffold-centos7.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tikv + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: centos7/Dockerfile + target: builder + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true + tagPolicy: + customTemplate: + template: "{{ .SHA }}-centos7" + components: + - name: SHA + gitCommit: + variant: Tags +profiles: + - name: devtoolset8 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-devtoolset8" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + DEVTOOLSET_VER: 8 + - name: devtoolset9 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-devtoolset9" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + DEVTOOLSET_VER: 9 + - name: devtoolset10 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "{{ .SHA }}-centos7-devtoolset10" + - op: add + path: /build/artifacts/0/docker/buildArgs + value: + DEVTOOLSET_VER: 10 diff --git a/dockerfiles/cd/builders/tikv/skaffold.yaml b/dockerfiles/cd/builders/tikv/skaffold.yaml new file mode 100644 index 00000000..2682c08b --- /dev/null +++ b/dockerfiles/cd/builders/tikv/skaffold.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=https://github.com/GoogleContainerTools/skaffold/raw/refs/heads/main/docs-v2/content/en/schemas/v4beta6.json +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder +build: + artifacts: + - image: tikv + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder-fips +build: + artifacts: + - image: tikv + platforms: [linux/amd64, linux/arm64] + docker: + dockerfile: fips.Dockerfile + target: builder + tagPolicy: + customTemplate: + template: "{{ .SHA }}-fips" + components: + - name: SHA + gitCommit: + variant: Tags + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + tryImportMissing: true