feat: add fips config (#157)

Signed-off-by: lijie <lijie@pingcap.com> Co-authored-by: wuhuizuo <wuhuizuo@126.com>
PingCAP-QE · Dec 15, 2023 · 77c2679 · 77c2679
1 parent 6dfbaa2
commit 77c2679
Show file tree

Hide file tree

Showing 24 changed files with 441 additions and 40 deletions.
diff --git a/.github/workflows/pull-cd-builder-images.yaml b/.github/workflows/pull-cd-builder-images.yaml
@@ -22,9 +22,8 @@ jobs:
       packages: write
 
     strategy:
-      max-parallel: 1
       matrix:
-        module: [builder-go, builder-others]
+        module: [builder-go, builder-others, builder-others-fips]
         go-profile: [go-1.21, go-1.20, go-1.19]
         builder-profile: [local-docker]
         platform: [linux/amd64, linux/arm64]
@@ -33,7 +32,10 @@ jobs:
             go-profile: go-1.20
           - module: builder-others
             go-profile: go-1.19
-
+          - module: builder-others-fips
+            go-profile: go-1.20
+          - module: builder-others-fips
+            go-profile: go-1.19
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4

diff --git a/.github/workflows/pull-cd-util-images.yaml b/.github/workflows/pull-cd-util-images.yaml
@@ -22,7 +22,6 @@ jobs:
       packages: write
 
     strategy:
-      max-parallel: 1
       matrix:
         module: [utils]
         builder-profile: [local-docker]

diff --git a/.github/workflows/pull-ci-runtime-images.yaml b/.github/workflows/pull-ci-runtime-images.yaml
@@ -22,7 +22,6 @@ jobs:
       packages: write
 
     strategy:
-      max-parallel: 1
       matrix:
         module: [base]
         platform: [linux/amd64, linux/arm64]

diff --git a/.github/workflows/pull-prod-runtime-images.yaml b/.github/workflows/pull-prod-runtime-images.yaml
@@ -22,8 +22,8 @@ jobs:
       packages: write
 
     strategy:
-      max-parallel: 1
       matrix:
+        module: [default, fips]
         builder-profile: [local-docker]
         platform: [linux/amd64, linux/arm64]
 
@@ -66,6 +66,7 @@ jobs:
             --build-concurrency 1 \
             --cache-artifacts \
             --default-repo ghcr.io/pingcap-qe/bases \
+            --module ${{ matrix.module }} \
             --platform ${{ matrix.platform }} \
             --profile ${{ matrix.builder-profile }} \
             --push=false
diff --git a/.github/workflows/release-cd-builder-images.yaml b/.github/workflows/release-cd-builder-images.yaml
@@ -19,17 +19,19 @@ jobs:
       packages: write
 
     strategy:
-      max-parallel: 1
       matrix:
-        module: [builder-go, builder-others]
+        module: [builder-go, builder-others, builder-others-fips]
         go-profile: [go-1.21, go-1.20, go-1.19]
         builder-profile: [local-docker]
         exclude:
           - module: builder-others
             go-profile: go-1.20
           - module: builder-others
             go-profile: go-1.19
-
+          - module: builder-others-fips
+            go-profile: go-1.20
+          - module: builder-others-fips
+            go-profile: go-1.19
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4

diff --git a/.github/workflows/release-cd-util-images.yaml b/.github/workflows/release-cd-util-images.yaml
@@ -19,7 +19,6 @@ jobs:
       packages: write
 
     strategy:
-      max-parallel: 1
       matrix:
         module: [utils]
         builder-profile: [local-docker]

diff --git a/.github/workflows/release-ci-runtime-images.yaml b/.github/workflows/release-ci-runtime-images.yaml
@@ -19,7 +19,6 @@ jobs:
       packages: write
 
     strategy:
-      max-parallel: 1
       matrix:
         module: [base]
         builder-profile: [local-docker]

diff --git a/.github/workflows/release-prod-runtime-images.yaml b/.github/workflows/release-prod-runtime-images.yaml
@@ -19,8 +19,8 @@ jobs:
       packages: write
 
     strategy:
-      max-parallel: 1
       matrix:
+        module: [default, fips]
         builder-profile: [local-docker]
 
     steps:
@@ -62,4 +62,5 @@ jobs:
             --build-concurrency 1 \
             --cache-artifacts \
             --default-repo ghcr.io/pingcap-qe/bases \
+            --module ${{ matrix.module }} \
             --profile ${{ matrix.builder-profile }}
diff --git a/dockerfiles-multi-stages/dumpling/Dockerfile b/dockerfiles-multi-stages/dumpling/Dockerfile
diff --git a/dockerfiles/bases/ng-monitoring-base/Dockerfile b/dockerfiles/bases/ng-monitoring-base/Dockerfile
@@ -1,6 +1,3 @@
-# hub.pingcap.net/bases/ng-monitoring-base:v1.7.0
-# hub.pingcap.net/bases/ng-monitoring-base:v1.7
-# hub.pingcap.net/bases/ng-monitoring-base:v1
 ARG PINGCAP_BASE
 FROM $PINGCAP_BASE
 RUN dnf install perl-interpreter -y && \

diff --git a/dockerfiles/bases/pd-base/Dockerfile b/dockerfiles/bases/pd-base/Dockerfile
@@ -1,6 +1,3 @@
-# hub.pingcap.net/bases/pd-base:v1.7.1
-# hub.pingcap.net/bases/pd-base:v1.7
-# hub.pingcap.net/bases/pd-base:v1
 ARG PINGCAP_BASE
 FROM $PINGCAP_BASE
 RUN dnf install bind-utils wget jq perl-interpreter -y && \

diff --git a/dockerfiles/bases/pingcap-base/Dockerfile b/dockerfiles/bases/pingcap-base/Dockerfile
@@ -1,6 +1,3 @@
-# hub.pingcap.net/bases/pingcap-base:v1.7.0
-# hub.pingcap.net/bases/pingcap-base:v1.7
-# hub.pingcap.net/bases/pingcap-base:v1
 FROM rockylinux:9.3.20231119
 COPY --from=busybox:1.36.1 /bin/busybox /bin/busybox
 RUN dnf upgrade -y python3 less && \

diff --git a/dockerfiles/bases/skaffold.yaml b/dockerfiles/bases/skaffold.yaml
@@ -1,6 +1,8 @@
 # Ref: https://skaffold.dev/docs/builders/builder-types/docker/#dockerfile-in-cluster-with-kaniko
 apiVersion: skaffold/v4beta6
 kind: Config
+metadata:
+  name: default
 build:
   artifacts:
     - image: pingcap-base      
@@ -102,3 +104,43 @@ profiles:
       - op: move
         from: /build/artifacts/6/kaniko
         path: /build/artifacts/6/docker
+---
+apiVersion: skaffold/v4beta6
+kind: Config
+metadata:
+  name: fips
+build:
+  artifacts:
+    - image: tikv-base
+      platforms: [linux/amd64, linux/arm64]
+      kaniko:
+        dockerfile: tikv-base/fips.Dockerfile
+        cache: {}
+  tagPolicy:
+    customTemplate:
+      template: "v1.8.0-fips"
+  cluster:
+    concurrency: 0
+    randomDockerConfigSecret: false
+    randomPullSecret: false
+    dockerConfig:
+      secretName: hub-pingcap-net
+    resources:
+      requests:
+        cpu: "1"
+        memory: 4Gi
+      limits:
+        cpu: "2"
+        memory: 8Gi
+profiles:
+  - name: local-docker
+    build:
+      local:
+        useDockerCLI: true
+        useBuildkit: true        
+        concurrency: 0
+        tryImportMissing: true
+    patches:
+      - op: move
+        from: /build/artifacts/0/kaniko
+        path: /build/artifacts/0/docker
diff --git a/dockerfiles/bases/tidb-base/Dockerfile b/dockerfiles/bases/tidb-base/Dockerfile
@@ -1,6 +1,3 @@
-# hub.pingcap.net/bases/tidb-base:v1.7.0
-# hub.pingcap.net/bases/tidb-base:v1.7
-# hub.pingcap.net/bases/tidb-base:v1
 ARG PINGCAP_BASE
 FROM $PINGCAP_BASE
 RUN dnf install --allowerasing -y curl wget && dnf clean all
diff --git a/dockerfiles/bases/tiflash-base/Dockerfile b/dockerfiles/bases/tiflash-base/Dockerfile
@@ -1,6 +1,3 @@
-# hub.pingcap.net/bases/tiflash-base:v1.7.0
-# hub.pingcap.net/bases/tiflash-base:v1.7
-# hub.pingcap.net/bases/tiflash-base:v1
 ARG PINGCAP_BASE
 FROM $PINGCAP_BASE
 RUN dnf install --allowerasing -y wget && dnf clean all
diff --git a/dockerfiles/bases/tikv-base/Dockerfile b/dockerfiles/bases/tikv-base/Dockerfile
@@ -1,6 +1,3 @@
-# hub.pingcap.net/bases/tikv-base:v1.7.0
-# hub.pingcap.net/bases/tikv-base:v1.7
-# hub.pingcap.net/bases/tikv-base:v1
 ARG PINGCAP_BASE
 FROM $PINGCAP_BASE
 # wget is requested by operator

diff --git a/dockerfiles/bases/tikv-base/fips.Dockerfile b/dockerfiles/bases/tikv-base/fips.Dockerfile
@@ -0,0 +1,6 @@
+ARG PINGCAP_BASE=ghcr.io/pingcap-qe/bases/pingcap-base:v1.8.0
+FROM $PINGCAP_BASE
+# wget is requested by operator
+RUN dnf install -y tzdata wget openssl && dnf clean all
+ENV TZ=/etc/localtime \
+    TZDIR=/usr/share/zoneinfo
diff --git a/dockerfiles/bases/tools-base/Dockerfile b/dockerfiles/bases/tools-base/Dockerfile
@@ -1,6 +1,3 @@
-# hub.pingcap.net/bases/tools-base:v1.7.0
-# hub.pingcap.net/bases/tools-base:v1.7
-# hub.pingcap.net/bases/tools-base:v1
 ARG PINGCAP_BASE
 FROM $PINGCAP_BASE
 RUN dnf install -y bind-utils wget nc && dnf clean all
diff --git a/dockerfiles/cd/builders/skaffold.yaml b/dockerfiles/cd/builders/skaffold.yaml
@@ -107,22 +107,66 @@ kind: Config
 metadata:
   name: builder-others
 build:
-  artifacts:  
-    - image: tiflash
+  artifacts:
+    - image: tikv
       platforms: [linux/amd64, linux/arm64]
       kaniko:
-        dockerfile: tiflash/Dockerfile
+        dockerfile: tikv/Dockerfile
         cache: {}
-        target: builder  
+        target: builder
+    # - image: tiflash
+    #   platforms: [linux/amd64, linux/arm64]
+    #   kaniko:
+    #     dockerfile: tiflash/Dockerfile
+    #     cache: {}
+    #     target: builder  
+  tagPolicy:
+    customTemplate:
+      template: "{{ .SHA }}"
+      components:
+        - name: SHA
+          gitCommit:
+            variant: AbbrevCommitSha
+  cluster:
+    concurrency: 0
+    randomDockerConfigSecret: false
+    randomPullSecret: false
+    dockerConfig:
+      secretName: hub-pingcap-net-ee
+    resources:
+      requests:
+        cpu: "1"
+        memory: 2Gi
+      limits:
+        cpu: "2"
+        memory: 4Gi
+profiles:
+  - name: local-docker
+    build:
+      local:
+        useDockerCLI: true
+        useBuildkit: true
+        concurrency: 0
+        tryImportMissing: true
+    patches:
+      - { op: move, from: /build/artifacts/0/kaniko, path: /build/artifacts/0/docker }
+      # - { op: move, from: /build/artifacts/1/kaniko, path: /build/artifacts/1/docker }
+---
+apiVersion: skaffold/v4beta6
+kind: Config
+metadata:
+  name: builder-others-fips
+build:
+  artifacts:
     - image: tikv
       platforms: [linux/amd64, linux/arm64]
       kaniko:
-        dockerfile: tikv/Dockerfile
+        dockerfile: tikv/fips.Dockerfile
         cache: {}
         target: builder
   tagPolicy:
     customTemplate:
-      template: "{{ .SHA }}"
+      template: "{{ .SHA }}-fips"
       components:
         - name: SHA
           gitCommit:
@@ -150,4 +194,3 @@ profiles:
         tryImportMissing: true
     patches:
       - { op: move, from: /build/artifacts/0/kaniko, path: /build/artifacts/0/docker }
-      - { op: move, from: /build/artifacts/1/kaniko, path: /build/artifacts/1/docker }
diff --git a/dockerfiles/cd/builders/tidb-dashboard/Dockerfile b/dockerfiles/cd/builders/tidb-dashboard/Dockerfile
@@ -0,0 +1,27 @@
+FROM centos:7.9.2009 as builder
+
+RUN yum -y update
+RUN yum -y groupinstall "Development Tools"
+
+# Install golang.
+ENV PATH /usr/local/go/bin:$PATH
+RUN export ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \\
+    export GO_VERSION=1.19.5 && \\
+    curl -OL https://golang.org/dl/go$GO_VERSION.linux-$ARCH.tar.gz && \\
+    tar -C /usr/local/ -xzf go$GO_VERSION.linux-$ARCH.tar.gz && \\
+    rm -f go$GO_VERSION.linux-$ARCH.tar.gz
+ENV GOROOT /usr/local/go
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:$PATH
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+
+# Install nodejs.
+RUN curl -fsSL https://rpm.nodesource.com/setup_16.x | bash -
+RUN yum -y install nodejs
+RUN npm install -g pnpm@7.30.5
+
+# Install java.
+RUN yum -y install java-11-openjdk
+
+RUN mkdir -p /go/src/github.com/pingcap/tidb-dashboard/ui
+WORKDIR /go/src/github.com/pingcap/tidb-dashboard
diff --git a/dockerfiles/cd/builders/tikv/fips.Dockerfile b/dockerfiles/cd/builders/tikv/fips.Dockerfile
@@ -0,0 +1,55 @@
+# build requires:
+#   - docker >= v20.10
+#
+# build steps:
+#   - git clone --recurse-submodules --branch feature/release-6.5-fips https://github.com/tikv/tikv.git tikv
+#   - docker build -t tikv -f Dockerfile ./tikv
+
+########### stage: Builder
+FROM rockylinux:9.3.20231119 as builder
+
+# install packages.
+RUN dnf install -y \
+      openssl-devel \
+      gcc \
+      gcc-c++ \
+      make \
+      cmake \
+      perl \
+      git \
+      findutils \
+      curl \
+      python3 --allowerasing && \
+    dnf --enablerepo=crb install -y \
+      libstdc++-static && \
+    dnf clean all
+
+# install protoc.
+# renovate: datasource=github-release depName=protocolbuffers/protobuf
+ARG PROTOBUF_VER=v3.15.8
+RUN FILE=$([ "$(arch)" = "aarch64" ] && echo "protoc-${PROTOBUF_VER#?}-linux-aarch_64.zip" || echo "protoc-${PROTOBUF_VER#?}-linux-$(arch).zip"); \
+    curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/${PROTOBUF_VER}/${FILE}" && unzip "$FILE" -d /usr/local/ && rm -f "$FILE"
+
+# Install Rustup
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s - -y --default-toolchain none
+ENV PATH /root/.cargo/bin/:$PATH
+
+########### stage: Buiding
+FROM builder as building
+COPY . /tikv
+RUN --mount=type=cache,target=/tikv/target \
+    source /opt/rh/devtoolset-8/enable && \
+    ENABLE_FIPS=1 \
+    ROCKSDB_SYS_STATIC=1 \
+    make dist_release -C /tikv
+RUN /tikv/bin/tikv-server --version
+
+########### stage: Final image
+FROM ghcr.io/pingcap-qe/bases/tikv-base:v1.8.0
+
+ENV MALLOC_CONF="prof:true,prof_active:false"
+COPY --from=building /tikv/bin/tikv-server  /tikv-server
+COPY --from=building /tikv/bin/tikv-ctl     /tikv-ctl
+
+EXPOSE 20160
+ENTRYPOINT ["/tikv-server"]
diff --git a/dockerfiles/products/tidb-dashboard.Dockerfile b/dockerfiles/products/tidb-dashboard.Dockerfile
@@ -0,0 +1,5 @@
+ARG BASE_IMG=hub.pingcap.net/bases/pd-base:v1.8.0
+FROM $BASE_IMG
+COPY tidb-dashboard /tidb-dashboard
+EXPOSE 12333
+ENTRYPOINT ["/tidb-dashboard"]
diff --git a/dockerfiles/products/tikv-fips.Dockerfile b/dockerfiles/products/tikv-fips.Dockerfile
@@ -0,0 +1,7 @@
+ARG BASE_IMG=hub.pingcap.net/bases/tikv-base:v1.8.0-fips
+FROM $BASE_IMG
+COPY tikv-server /tikv-server
+COPY tikv-ctl /tikv-ctl
+ENV MALLOC_CONF="prof:true,prof_active:false"
+EXPOSE 20160
+ENTRYPOINT ["/tikv-server"]