diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 6935506d..00000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,217 +0,0 @@ -name: Skaffold - test -on: - pull_request: - branches: - - main - paths: - - "**/Dockerfile" - - "**/*.Dockerfile" - - "**/skaffold.yaml" - - .github/workflows/ci.yml - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # JOB to run change detection - dockerfiles-changes: - name: detect changes for Dockerfiles. - runs-on: ubuntu-latest - # Required permissions - permissions: - pull-requests: read - outputs: - # Expose matched filters as job 'packages' output variable - packages-prod-bases: ${{ steps.filter-product-bases.outputs.changes }} - packages-ci-bases: ${{ steps.filter-ci-bases.outputs.changes }} - packages-ci-utils: ${{ steps.filter-ci-utils.outputs.changes }} - steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter-product-bases - with: - filters: | - bases: dockerfiles/bases/** - - uses: dorny/paths-filter@v2 - id: filter-ci-bases - with: - filters: | - base: dockerfiles/ci/base/** - release-build-base: dockerfiles/ci/release-build-base/** - - uses: dorny/paths-filter@v2 - id: filter-ci-utils - with: - filters: | - release-utils: dockerfiles/ci/release-utils/** - - build-product-bases: - name: Skaffold Build product base images - needs: dockerfiles-changes - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - strategy: - matrix: - no-use: ${{ fromJSON(needs.dockerfiles-changes.outputs.packages-prod-bases) }} - builder-profile: [local-docker] - platform: [linux/amd64, linux/arm64] - - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup skaffold - run: | - curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \ - sudo install skaffold /usr/local/bin/ - - - name: Cache layers - uses: actions/cache@v3 - with: - path: "~/.skaffold/cache" - key: skaffold-${{ hashFiles('**/cache') }} - restore-keys: | - skaffold- - - - name: Build images - working-directory: dockerfiles/bases - run: | - skaffold build \ - --push=false \ - --platform ${{ matrix.platform }} \ - --profile ${{ matrix.builder-profile }} \ - --default-repo ghcr.io/pingcap-qe/bases - - build-ci-bases: - name: Skaffold build CI base images - needs: dockerfiles-changes - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - strategy: - matrix: - module: ${{ fromJSON(needs.dockerfiles-changes.outputs.packages-ci-bases) }} - go-profile: [go-1.21, go-1.20, go-1.19] - builder-profile: [local-docker] - platform: [linux/amd64, linux/arm64] - - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup skaffold - run: | - curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \ - sudo install skaffold /usr/local/bin/ - - - name: Cache layers - uses: actions/cache@v3 - with: - path: "~/.skaffold/cache" - key: skaffold-${{ hashFiles('**/cache') }} - restore-keys: | - skaffold- - - - name: Build images - working-directory: dockerfiles/ci - run: | - skaffold build \ - --push=false \ - --platform ${{ matrix.platform }} \ - --module ${{ matrix.module }} \ - --profile ${{ matrix.go-profile }},${{ matrix.builder-profile }} \ - --default-repo ghcr.io/pingcap-qe/ci - - build-ci-utils: - name: Skaffold Build ci util images - needs: dockerfiles-changes - runs-on: ubuntu-latest - - permissions: - contents: read - packages: write - - strategy: - matrix: - no-use: ${{ fromJSON(needs.dockerfiles-changes.outputs.packages-ci-utils) }} - builder-profile: [local-docker] - platform: [linux/amd64, linux/arm64] - - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup skaffold - run: | - curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \ - sudo install skaffold /usr/local/bin/ - - - name: Cache layers - uses: actions/cache@v3 - with: - path: "~/.skaffold/cache" - key: skaffold-${{ hashFiles('**/cache') }} - restore-keys: | - skaffold- - - - name: Build images - working-directory: dockerfiles/ci - run: | - skaffold build \ - --push=false \ - --platform ${{ matrix.platform }} \ - --module release-utils \ - --profile ${{ matrix.builder-profile }} \ - --default-repo ghcr.io/pingcap-qe/ci diff --git a/.github/workflows/pull-cd-build-base-images.yaml b/.github/workflows/pull-cd-build-base-images.yaml new file mode 100644 index 00000000..deb042e8 --- /dev/null +++ b/.github/workflows/pull-cd-build-base-images.yaml @@ -0,0 +1,81 @@ +name: Pull test for CD builder images +on: + pull_request: + branches: + - main + paths: + - "dockerfiles/cd/**/Dockerfile" + - "dockerfiles/cd/**/*.Dockerfile" + - "dockerfiles/cd/skaffold.yaml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + skaffold: + name: build images with skaffold + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + strategy: + matrix: + builder-profile: [local-docker] + platform: [linux/amd64, linux/arm64] + # go-profile: [go-1.21, go-1.20, go-1.19] + # module: [builder-go, builder-others] + go-profile: [go-1.21] + module: [builder-go] + exclude: + - module: builder-others + go-profile: go-1.20 + - module: builder-others + go-profile: go-1.19 + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + + - name: Cache layers + uses: actions/cache@v3 + with: + path: "~/.skaffold/cache" + key: skaffold/${{ matrix.platform }}/cd/${{ matrix.module }}/${{ matrix.go-profile }}-${{ github.sha }} + restore-keys: | + skaffold/${{ matrix.platform }}/cd/${{ matrix.module }}/${{ matrix.go-profile }}-${{ github.sha }} + skaffold/${{ matrix.platform }}/cd/${{ matrix.module }}/${{ matrix.go-profile }}- + skaffold/${{ matrix.platform }}/cd/${{ matrix.module }}/ + + - name: Build images + working-directory: dockerfiles/cd + run: | + skaffold build \ + --push=false \ + --build-concurrency 1 \ + --platform ${{ matrix.platform }} \ + --module ${{ matrix.module }} \ + --profile ${{ matrix.go-profile }},${{ matrix.builder-profile }} \ + --default-repo ghcr.io/pingcap-qe/cd/builders diff --git a/.github/workflows/pull-ci-release-utils-images.yaml b/.github/workflows/pull-ci-release-utils-images.yaml new file mode 100644 index 00000000..c2654110 --- /dev/null +++ b/.github/workflows/pull-ci-release-utils-images.yaml @@ -0,0 +1,71 @@ +name: Pull test for CI utils images +on: + pull_request: + branches: + - main + paths: + - "dockerfiles/ci/release-utils/**/Dockerfile" + - "dockerfiles/ci/release-utils/**/*.Dockerfile" + - "dockerfiles/ci/skaffold.yaml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + skaffold: + name: build images with skaffold + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + strategy: + matrix: + builder-profile: [local-docker] + platform: [linux/amd64, linux/arm64] + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + + - name: Cache layers + uses: actions/cache@v3 + with: + path: "~/.skaffold/cache" + key: skaffold/${{ matrix.platform }}/release-utils-${{ github.sha }} + restore-keys: | + skaffold/${{ matrix.platform }}/release-utils-${{ github.sha }} + skaffold/${{ matrix.platform }}/release-utils- + + - name: Build images + working-directory: dockerfiles/ci + run: | + skaffold build \ + --cache-artifacts \ + --push=false \ + --platform ${{ matrix.platform }} \ + --module release-utils \ + --profile ${{ matrix.builder-profile }} \ + --default-repo ghcr.io/pingcap-qe/ci diff --git a/.github/workflows/pull-ci-runtime-base-images.yaml b/.github/workflows/pull-ci-runtime-base-images.yaml new file mode 100644 index 00000000..bef6ce98 --- /dev/null +++ b/.github/workflows/pull-ci-runtime-base-images.yaml @@ -0,0 +1,73 @@ +name: Pull test for CI base images +on: + pull_request: + branches: + - main + paths: + - "dockerfiles/ci/base/**/Dockerfile" + - "dockerfiles/ci/base/**/*.Dockerfile" + - "dockerfiles/ci/skaffold.yaml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + skaffold: + name: build images with skaffold + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + strategy: + matrix: + go-profile: [go-1.21, go-1.20, go-1.19] + builder-profile: [local-docker] + platform: [linux/amd64, linux/arm64] + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + + - name: Cache layers + uses: actions/cache@v3 + with: + path: "~/.skaffold/cache" + key: skaffold/${{ matrix.platform }}/ci-base-${{ matrix.go-profile }}-${{ github.sha }} + restore-keys: | + skaffold/${{ matrix.platform }}/ci-base-${{ matrix.go-profile }}-${{ github.sha }} + skaffold/${{ matrix.platform }}/ci-base-${{ matrix.go-profile }}- + skaffold/${{ matrix.platform }}/ci-base- + + - name: Build images + working-directory: dockerfiles/ci + run: | + skaffold build \ + --cache-artifacts \ + --push=false \ + --platform ${{ matrix.platform }} \ + --module base \ + --profile ${{ matrix.go-profile }},${{ matrix.builder-profile }} \ + --default-repo ghcr.io/pingcap-qe/ci diff --git a/.github/workflows/pull-prod-runtime-base-images.yaml b/.github/workflows/pull-prod-runtime-base-images.yaml new file mode 100644 index 00000000..b0fa4151 --- /dev/null +++ b/.github/workflows/pull-prod-runtime-base-images.yaml @@ -0,0 +1,71 @@ +name: Pull test for building product base images +on: + pull_request: + branches: + - main + paths: + - "dockerfiles/bases/**/Dockerfile" + - "dockerfiles/bases/**/*.Dockerfile" + - "dockerfiles/bases/skaffold.yaml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + skaffold: + name: build images with skaffold + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + strategy: + matrix: + builder-profile: [local-docker] + platform: [linux/amd64, linux/arm64] + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup skaffold + run: | + curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/v2.8.0/skaffold-linux-amd64 && \ + sudo install skaffold /usr/local/bin/ + + - name: Cache layers + uses: actions/cache@v3 + with: + path: "~/.skaffold/cache" + key: skaffold/${{ matrix.platform }}/prod-bases-${{ github.sha }} + restore-keys: | + skaffold/${{ matrix.platform }}/prod-bases-${{ github.sha }} + skaffold/${{ matrix.platform }}/prod-bases- + + - name: Build images + working-directory: dockerfiles/bases + run: | + skaffold build \ + --cache-artifacts \ + --push=false \ + --platform ${{ matrix.platform }} \ + --profile ${{ matrix.builder-profile }} \ + --default-repo ghcr.io/pingcap-qe/bases + diff --git a/Justfile b/Justfile index 3dcaff83..28c54766 100644 --- a/Justfile +++ b/Justfile @@ -1,8 +1,10 @@ -msb-tikv: (_msb "tikv" "https://github.com/tikv/tikv.git" "master") - docker run --rm tikv:latest --version +msb-tikv: (_clone "tikv" "https://github.com/tikv/tikv.git" "master") + docker build --load -t localhost/tikv:local-build -f dockerfiles/cd/tikv/tikv/Dockerfile ../tikv + docker run --rm localhost/tikv:local-build --version -msb-tiflash: (_msb "tiflash" "https://github.com/pingcap/tiflash" "master") - docker run --rm --entrypoint=/tiflash/tiflash tiflash:latest version +msb-tiflash: (_clone "tiflash" "https://github.com/pingcap/tiflash" "master") + docker build --load -t localhost/tiflash:local-build -f dockerfiles/cd/pingcap/tiflash/Dockerfile ../tiflash + docker run --rm --entrypoint=/tiflash/tiflash localhost/tiflash:local-build version msb-dm: (_clone "tiflow" "https://github.com/pingcap/tiflow.git" "master") docker build --load -t localhost/dm:local-build -f dockerfiles/cd/pingcap/tiflow/Dockerfile --target final-dm ../tiflow diff --git a/dockerfiles-multi-stages/tikv/Dockerfile b/dockerfiles-multi-stages/tikv/Dockerfile deleted file mode 100644 index 1af4a3a1..00000000 --- a/dockerfiles-multi-stages/tikv/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM rockylinux:8 as builder -RUN yum groupinstall -y "Development Tools" -RUN yum --enablerepo=powertools install -y git make cmake gawk llvm protobuf-compiler libstdc++-static python3 -RUN ln -s /usr/bin/python3 /usr/bin/python -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y --default-toolchain none -ENV PATH /root/.cargo/bin/:$PATH -WORKDIR /tikv -ARG GIT_FALLBACK="unknown git info" -ARG GIT_HASH=${GIT_FALLBACK} -ARG GIT_TAG=${GIT_FALLBACK} -ARG GIT_BRANCH=${GIT_FALLBACK} -ENV TIKV_BUILD_GIT_HASH=${GIT_HASH} -ENV TIKV_BUILD_GIT_TAG=${GIT_TAG} -ENV TIKV_BUILD_GIT_BRANCH=${GIT_BRANCH} -ADD . . -RUN ROCKSDB_SYS_STATIC=1 make dist_release -RUN bin/tikv-server --version - -FROM rockylinux:9.2 -COPY --from=busybox:1.36.1 /bin/busybox /bin/busybox - -RUN dnf install -y tzdata -ENV TZ=/etc/localtime -ENV TZDIR=/usr/share/zoneinfo - -COPY --from=builder /tikv/bin/tikv-server /tikv-server -COPY --from=builder /tikv/bin/tikv-ctl /tikv-ctl -ENV MALLOC_CONF="prof:true,prof_active:false" -EXPOSE 20160 -ENTRYPOINT ["/tikv-server"] \ No newline at end of file diff --git a/dockerfiles/cd/pingcap/tiflash/Dockerfile b/dockerfiles/cd/pingcap/tiflash/Dockerfile new file mode 100644 index 00000000..60067485 --- /dev/null +++ b/dockerfiles/cd/pingcap/tiflash/Dockerfile @@ -0,0 +1,42 @@ +# build requires: +# - docker >= v20.10 +# +# build steps: +# - git clone --recurse-submodules https://github.com/tikv/tikv.git tikv +# - docker build -t tikv -f Dockerfile ./tikv + +########### stage: Builder +FROM centos:7.9.2009 as builder + +LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" + +# renovate: datasource=github-tags depName=pingcap/tiflash +ARG TIFLASH_VER=v7.4.0 +RUN FILE=$([ "$(arch)" = "aarch64" ] && echo "bake_llvm_base_aarch64.sh" || echo "bake_llvm_base_amd64.sh"); \ + curl -sSL https://github.com/pingcap/tiflash/archive/refs/tags/${TIFLASH_VER}.tar.gz -o tiflash.tar.gz && \ + tar -zxf tiflash.tar.gz "tiflash-${TIFLASH_VER#?}/release-centos7-llvm/dockerfiles/misc" && \ + pushd "tiflash-${TIFLASH_VER#?}/release-centos7-llvm/dockerfiles/misc" && \ + chmod +x * && ./${FILE} && \ + popd && \ + rm -rf tiflash.tar.gz "tiflash-${TIFLASH_VER#?}" + +ENV PATH="/opt/cmake/bin:${PATH}:/usr/local/go/bin:/root/.cargo/bin" +ENV CC=clang +ENV CXX=clang++ +ENV LD=ld.lld + +########### stage: Buiding +FROM builder as building + +ADD . /tiflash +RUN /tiflash/release-centos7-llvm/scripts/build-release.sh +RUN mkdir output && mv release-centos7-llvm/tiflash output/tiflash +RUN output/tiflash/tiflash version + +########### stage: Final image +FROM ghcr.io/pingcap-qe/bases/tiflash-base:v1.7.1 + +ENV LD_LIBRARY_PATH /tiflash +COPY --from=building /tiflash/output/tiflash /tiflash + +ENTRYPOINT ["/tiflash/tiflash", "server"] diff --git a/dockerfiles/cd/skaffold.yaml b/dockerfiles/cd/skaffold.yaml new file mode 100644 index 00000000..33ba24d0 --- /dev/null +++ b/dockerfiles/cd/skaffold.yaml @@ -0,0 +1,159 @@ +# Ref: https://skaffold.dev/docs/builders/builder-types/docker/#dockerfile-in-cluster-with-kaniko +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder-go +build: + artifacts: + - image: tidb + platforms: [linux/amd64, linux/arm64] + kaniko: + dockerfile: pingcap/tidb/Dockerfile + cache: {} + target: builder + - image: tiflow + platforms: [linux/amd64, linux/arm64] + kaniko: + dockerfile: pingcap/tiflow/Dockerfile + cache: {} + target: builder + - image: pd + platforms: [linux/amd64, linux/arm64] + kaniko: + dockerfile: tikv/pd/Dockerfile + cache: {} + target: builder + tagPolicy: + customTemplate: + template: "v{{ .DATE }}-{{ .SHA }}" + components: + - name: DATE + dateTime: + format: "20060102" + timezone: "UTC" + - name: SHA + gitCommit: + variant: AbbrevCommitSha + cluster: + concurrency: 0 + randomDockerConfigSecret: false + randomPullSecret: false + dockerConfig: + secretName: hub-pingcap-net-ee + resources: + limits: + cpu: "4" + memory: 8Gi +profiles: + - name: local-docker + build: + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + patches: + - { op: move, from: /build/artifacts/0/kaniko, path: /build/artifacts/0/docker } + - { op: move, from: /build/artifacts/1/kaniko, path: /build/artifacts/1/docker } + - { op: move, from: /build/artifacts/2/kaniko, path: /build/artifacts/2/docker } + - name: go-1.21 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "v{{ .DATE }}-{{ .SHA }}-go1.21" + - op: add + path: /build/artifacts/0/kaniko/buildArgs + value: &value-go1-21 + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.21.4 + - op: add + path: /build/artifacts/1/kaniko/buildArgs + value: &value-go1-21 + - op: add + path: /build/artifacts/2/kaniko/buildArgs + value: &value-go1-21 + - name: go-1.20 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "v{{ .DATE }}-{{ .SHA }}-go1.20" + - op: add + path: /build/artifacts/0/kaniko/buildArgs + value: &value-go1-20 + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.20.10 + - op: add + path: /build/artifacts/1/kaniko/buildArgs + value: &value-go1-20 + - op: add + path: /build/artifacts/2/kaniko/buildArgs + value: &value-go1-20 + - name: go-1.19 + patches: + - op: replace + path: /build/tagPolicy/customTemplate/template + value: "v{{ .DATE }}-{{ .SHA }}-go1.19" + - op: add + path: /build/artifacts/0/kaniko/buildArgs + value: &value-go1-19 + # renovate: datasource=docker depName=golang + GOLANG_VERSION: 1.19.12 + - op: add + path: /build/artifacts/1/kaniko/buildArgs + value: &value-go1-19 + - op: add + path: /build/artifacts/2/kaniko/buildArgs + value: &value-go1-19 +--- +apiVersion: skaffold/v4beta6 +kind: Config +metadata: + name: builder-others +build: + artifacts: + - image: tiflash + platforms: [linux/amd64, linux/arm64] + kaniko: + dockerfile: pingcap/tiflash/Dockerfile + cache: {} + target: builder + - image: tikv + platforms: [linux/amd64, linux/arm64] + kaniko: + dockerfile: tikv/tikv/Dockerfile + cache: {} + target: builder + tagPolicy: + customTemplate: + template: "v{{ .DATE }}-{{ .SHA }}" + components: + - name: DATE + dateTime: + format: "20060102" + timezone: "UTC" + - name: SHA + gitCommit: + variant: AbbrevCommitSha + cluster: + concurrency: 0 + randomDockerConfigSecret: false + randomPullSecret: false + dockerConfig: + secretName: hub-pingcap-net-ee + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "2" + memory: 4Gi +profiles: + - name: local-docker + build: + local: + useDockerCLI: true + useBuildkit: true + concurrency: 0 + patches: + - { op: move, from: /build/artifacts/0/kaniko, path: /build/artifacts/0/docker } + - { op: move, from: /build/artifacts/1/kaniko, path: /build/artifacts/1/docker } diff --git a/dockerfiles/cd/tikv/tikv/Dockerfile b/dockerfiles/cd/tikv/tikv/Dockerfile new file mode 100644 index 00000000..8e8c470f --- /dev/null +++ b/dockerfiles/cd/tikv/tikv/Dockerfile @@ -0,0 +1,45 @@ +# build requires: +# - docker >= v20.10 +# +# build steps: +# - git clone --recurse-submodules https://github.com/tikv/tikv.git tikv +# - docker build -t tikv -f Dockerfile ./tikv + +########### stage: Builder +FROM centos:7.9.2009 as builder +# install packages. +RUN yum update --nogpgcheck -y && \ + yum install --nogpgcheck -y epel-release centos-release-scl deltarpm && \ + yum update --nogpgcheck -y && \ + yum install -y git devtoolset-8 perl cmake3 && \ + yum clean all && \ + ln -s /usr/bin/cmake3 /usr/bin/cmake + +# install protoc. +# renovate: datasource=github-release depName=protocolbuffers/protobuf +ARG PROTOBUF_VER=v3.15.8 +RUN FILE=$([ "$(arch)" = "aarch64" ] && echo "protoc-${PROTOBUF_VER#?}-linux-aarch_64.zip" || echo "protoc-${PROTOBUF_VER#?}-linux-$(arch).zip"); \ + curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/${PROTOBUF_VER}/${FILE}" && unzip "$FILE" -d /usr/local/ && rm -f "$FILE" + +# install rust toolchain +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y --default-toolchain none +ENV PATH /root/.cargo/bin/:$PATH + +########### stage: Buiding +FROM builder as building +COPY . /tikv +RUN --mount=type=cache,target=/tikv/target \ + source /opt/rh/devtoolset-8/enable && \ + ROCKSDB_SYS_STATIC=1 \ + make dist_release -C /tikv +RUN /tikv/bin/tikv-server --version + +########### stage: Final image +FROM ghcr.io/pingcap-qe/bases/tikv-base:v1.7.1 + +ENV MALLOC_CONF="prof:true,prof_active:false" +COPY --from=building /tikv/bin/tikv-server /tikv-server +COPY --from=building /tikv/bin/tikv-ctl /tikv-ctl + +EXPOSE 20160 +ENTRYPOINT ["/tikv-server"] diff --git a/dockerfiles/ci/release-build-base/tiflash/Dockerfile b/dockerfiles/ci/release-build-base/tiflash/Dockerfile deleted file mode 100644 index 3702075e..00000000 --- a/dockerfiles/ci/release-build-base/tiflash/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM centos:7.9.2009 - -LABEL org.opencontainers.image.authors "wuhui.zuo@pingcap.com" - -ARG ARCH=amd64 -ARG OS=linux -# renovate: datasource=github-tags depName=pingcap/tiflash -ARG TIFLASH_VER=v7.4.0 - -RUN curl -sSL https://github.com/pingcap/tiflash/archive/refs/tags/v${TIFLASH_VER}.tar.gz -o tiflash.tar.gz && \ - tar -zxf tiflash.tar.gz tiflash-${TIFLASH_VER}/release-centos7-llvm/dockerfiles/misc && \ - pushd tiflash-${TIFLASH_VER}/release-centos7-llvm/dockerfiles/misc && \ - chmod +x * && \ - if [ "$ARCH" = "arm64" ]; then \ - ./bake_llvm_base_aarch64.sh; \ - else \ - ./bake_llvm_base_amd64.sh; \ - fi && \ - popd && \ - rm -rf tiflash.tar.gz tiflash-${TIFLASH_VER} - -ENV PATH="/opt/cmake/bin:${PATH}:/usr/local/go/bin:/root/.cargo/bin" -ENV CC=clang -ENV CXX=clang++ -ENV LD=ld.lld - -# LIBRARY_PATH="/usr/local/lib/aarch64-unknown-linux-gnu/:${LIBRARY_PATH}" \ -# LD_LIBRARY_PATH="/usr/local/lib/aarch64-unknown-linux-gnu/:${LD_LIBRARY_PATH}" \ -# CPLUS_INCLUDE_PATH="/usr/local/include/aarch64-unknown-linux-gnu/c++/v1/:${CPLUS_INCLUDE_PATH}" \ -# OPENSSL_ROOT_DIR="/usr/local/opt/openssl" \