diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 32505efdf..353e66acd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,13 +8,11 @@ on: branches: - release/v* paths-ignore: - - '*.md' - '.gitignore' - 'CODEOWNERS' - 'LICENSE' pull_request: paths-ignore: - - '*.md' - '.gitignore' - 'CODEOWNERS' - 'LICENSE' @@ -43,21 +41,16 @@ jobs: git clone --depth 1 -b release/v2.9 https://github.com/rancher/rancher.git rancherDir cd rancherDir ./scripts/chart/build chart - sed -i -e 's/kubeVersion: < 1.29.0-0/kubeVersion: < 1.30.0-0/' build/chart/rancher/Chart.yaml tar cfz "${{ runner.temp }}/rancher.tgz" -C build/chart/rancher . popd - run : make ci name: ci -# - name: Build an image for integration testing -# run: ./scripts/package-for-ci -# env: -# ARCH: "${{ matrix.arch }}" -# SHA: "${{ github.sha }}" - - name : install K3d run : ./.github/workflows/scripts/install-k3d.sh + env: + K3D_VERSION: latest - name : setup cluster run : ./.github/workflows/scripts/setup-cluster.sh @@ -72,14 +65,10 @@ jobs: - name : start rancher run : ./.github/workflows/scripts/start-rancher.sh env: - ARCH: "${{ matrix.arch }}" CHART_PATH: "${{ runner.temp }}/rancher.tgz" RANCHER_IMAGE_TAG: "v2.9-head" VERSION: "2.9" - - name : debug show image_tag file - run : cat dist/image_tag - - name : get vars run : cat dist/image_tag >> $GITHUB_ENV @@ -89,5 +78,5 @@ jobs: ARCH: "${{ matrix.arch }}" CLUSTER_NAME: webhook IMAGE_REPO: rancher/webhook - IMAGE_TAG: "${{ env.TAG }}" + IMAGE_TAG: "${{ env.IMAGE_TAG }}" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 4760ea10a..514f8d083 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -3,7 +3,7 @@ name : Publish Webhook Images on: push: tags: - - "*" + - release/v* env: REGISTRY: docker.io diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2fa0f16de..08842f155 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,7 +3,7 @@ name: goreleaser on: push: tags: - - '*' + - release/v* permissions: contents: write @@ -22,12 +22,11 @@ jobs: - uses: actions/checkout@v4 with: fetch-tags: true - - uses: actions/setup-go@v5 + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 with: - go-version: 1.22 + go-version-file: 'go.mod' - name: Package release helm charts - run: make package-helm - - run: mkdir -p ./build/artifacts/ && mv -v ./dist/artifacts/ ./build/ + run: make package-helm && mkdir -p ./build/artifacts/ && mv -v ./dist/artifacts/ ./build/ - uses: goreleaser/goreleaser-action@v5 with: distribution: goreleaser diff --git a/.github/workflows/scripts/install-k3d.sh b/.github/workflows/scripts/install-k3d.sh index 51ed39f9f..6993f3312 100755 --- a/.github/workflows/scripts/install-k3d.sh +++ b/.github/workflows/scripts/install-k3d.sh @@ -1,17 +1,18 @@ #!/bin/bash -set -e -set -x +set -eu +REPO_URL=https://github.com/rancher/k3d K3D_URL=https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh -DEFAULT_K3D_VERSION=v5.4.6 install_k3d(){ - local k3dVersion=${K3D_VERSION:-${DEFAULT_K3D_VERSION}} - echo -e "Downloading k3d@${k3dVersion} see: ${K3D_URL}" - curl --silent --fail ${K3D_URL} | TAG=${k3dVersion} bash + if [ -z "${K3D_VERSION:-}" -o "${K3D_VERSION:-}" = "latest" ] ; then + K3D_VERSION=$(curl -Ls -o /dev/null -w %{url_effective} "${REPO_URL}/releases/latest" | grep -oE "[^/]+$") + fi + echo -e "Downloading k3d@${K3D_VERSION} from ${K3D_URL}" + curl --silent --fail ${K3D_URL} | TAG=${K3D_VERSION} bash } install_k3d -k3d version \ No newline at end of file +k3d version diff --git a/.github/workflows/scripts/integration-test-ci b/.github/workflows/scripts/integration-test-ci index 2481d00fc..d5552c89b 100755 --- a/.github/workflows/scripts/integration-test-ci +++ b/.github/workflows/scripts/integration-test-ci @@ -1,5 +1,5 @@ #!/bin/bash -set -eux +set -eu cd $(dirname $0)/../../.. DIST_DIR="$PWD"/dist @@ -45,7 +45,7 @@ echo "Uploading new webhook image" # Install the webhook chart we just built. upgrade_rancher_webhook() { - helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system \ + helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_CHART_VERSION}.tgz -n cattle-system \ --wait --timeout=120s --set image.repository="${IMAGE_REPO}" --set image.tag="${IMAGE_TAG}" --reuse-values --debug } @@ -68,7 +68,7 @@ set -e ./bin/rancher-webhook-integration.test -test.v -test.run IntegrationTest # Install the webhook chart with new ports. -helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system \ +helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_CHART_VERSION}.tgz -n cattle-system \ --wait --reuse-values --set port=443 # Test that the ports are set as expected and run a single integration test to verify the webhook is still accessible. @@ -78,5 +78,4 @@ helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tg # Scale down rancher-webhook so that we can run tests on the FailurePolicy. kubectl scale deploy rancher-webhook -n cattle-system --replicas=0 kubectl wait pods -l app=rancher-webhook --for=delete -n cattle-system -kubectl wait pods -l app=rancher-webhook --for=delete -n cattle-system ./bin/rancher-webhook-integration.test -test.v -test.run FailurePolicyTest diff --git a/.github/workflows/scripts/setup-cluster.sh b/.github/workflows/scripts/setup-cluster.sh index 8da39fc0f..2930e775e 100755 --- a/.github/workflows/scripts/setup-cluster.sh +++ b/.github/workflows/scripts/setup-cluster.sh @@ -57,12 +57,10 @@ wait_for_nodes(){ done } -k3d registry list --no-headers -o json | jq -r '.[].name' | xargs --no-run-if-empty k3d registry delete -k3d cluster delete $CLUSTER_NAME || true k3d registry create gha -p 42765 k3d cluster create $CLUSTER_NAME --servers 1 --agents 1 \ --registry-use gha:42765 \ - --image "docker.io/rancher/k3s:${K3S_VERSION}" --api-port 6550 + --image "rancher/k3s:${K3S_VERSION}" --api-port 6550 wait_for_nodes diff --git a/.github/workflows/scripts/start-rancher.sh b/.github/workflows/scripts/start-rancher.sh index 86e261211..1d8e23194 100755 --- a/.github/workflows/scripts/start-rancher.sh +++ b/.github/workflows/scripts/start-rancher.sh @@ -17,7 +17,7 @@ set -e helm repo update -helm upgrade --install cert-manager --namespace cert-manager cert-manager/cert-manager --set installCRDs=true --set "extraArgs[0]=--enable-certificate-owner-ref=true" --create-namespace --wait --timeout=10m +helm upgrade --install cert-manager --namespace cert-manager cert-manager/cert-manager --set installCRDs=true --create-namespace --wait --timeout=10m # kubectl get pods --namespace cert-manager kubectl rollout status --namespace cert-manager deploy/cert-manager --timeout 1m diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 62303aef6..9f5306abf 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -17,7 +17,7 @@ RUN if [ "${ARCH}" = "amd64" ]; then \ RUN GOBIN=/usr/local/bin go install github.com/golang/mock/mockgen@v1.6.0 -ENV DAPPER_ENV REPO TAG DRONE_TAG CROSS +ENV DAPPER_ENV REPO TAG CROSS ENV DAPPER_SOURCE /go/src/github.com/rancher/webhook/ ENV DAPPER_OUTPUT ./bin ./dist ENV DAPPER_DOCKER_SOCKET true diff --git a/package/Dockerfile b/package/Dockerfile index 111efe47d..b52dd3a86 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -11,4 +11,4 @@ COPY bin/webhook /usr/bin/ USER $user -CMD ["/usr/bin/webhook"] +CMD ["webhook"] diff --git a/scripts/integration-test b/scripts/integration-test index 3c73322cc..1fd719375 100755 --- a/scripts/integration-test +++ b/scripts/integration-test @@ -1,44 +1,83 @@ #!/bin/bash -set -exu +set -e +export KUBECONFIG= +export CATTLE_DEV_MODE=yes +export CATTLE_SERVER_URL="https://$(ip route get 8.8.8.8 | awk '{print $7}'):443" +export CATTLE_BOOTSTRAP_PASSWORD="admin" +export CATTLE_FEATURES="harvester=false" cd $(dirname $0)/../ -source ./scripts/try.sh +echo "Starting Rancher Server" +entrypoint.sh >./rancher.log 2>&1 & +RANCHER_PID=$! -# Wait for rancher to start up -try --delay 2 --max 30 --waitmsg "Waiting for rancher to start" --failmsg "No rancher here" kubectl rollout status --watch=true --timeout=10s -n cattle-system deploy/rancher -echo "Rancher deployed" +echo "Waiting for Rancher health check..." +while ! curl -sf http://localhost:80/healthz >/dev/null 2>&1; do + echo "Waiting for Rancher's /healthz endpoint to become available" + sleep 2 +done -# Wait for the rancher webhook to start up -try --delay 2 --max 30 --waitmsg "Waiting for rancher/webhook to be deployed" --failmsg "No webhook here" kubectl rollout status --watch=true --timeout=10s -n cattle-system deploy/rancher-webhook -echo "Webhook deployed" - -webhook_deployed() { - status=$(kubectl get apps.catalog.cattle.io -n cattle-system rancher-webhook -o jsonpath="{@.status.summary.state}") && [[ "$status" == "deployed" ]] -} +# Tail the rancher logs if rancher fails to deploy the webhook after 5 minutes. +bash -c "sleep 300 && echo 'Rancher has not deployed webhook after 5m tailing logs' && tail -f ./rancher.log" & +# Get PID of the tail command so we can kill it if needed +TAIL_PID=$! # Wait for Rancher to deploy rancher-webhook. -try --delay 2 --max 30 --waitmsg "Waiting for webhook to be deployed (2)" webhook_deployed +while ! kubectl rollout status -w -n cattle-system deploy/rancher-webhook >/dev/null 2>&1; do + echo "Waiting for rancher to deploy rancher-webhook..." + sleep 2 +done echo "Webhook deployed" -# Shut down the core rancher part, but leave the rest of the rancher environment running - -kubectl scale deploy rancher -n cattle-system --replicas=0 --timeout=10m -kubectl wait pods -l app=rancher-webhook --for=delete --namespace cattle-system --timeout=10m - -echo "Rancher has been stopped." +# After rancher deploys webhook kill the bash command running tail. +kill ${TAIL_PID} + +# Wait for helm operation to complete and save rancher-webhook release info before we kill rancher and the cluster. +while + status=$(kubectl get apps.catalog.cattle.io -n cattle-system rancher-webhook -o jsonpath="{@.status.summary.state}") + [[ "$status" != "deployed" ]] +do + echo "Waiting for helm operation to finish, current status $status" + sleep 2 +done + +# Kill Rancher since we only need the CRDs and the initial webhook values. +# We do not want Rancher to reconcile an older version of the webhook during test. +kill ${RANCHER_PID} + +echo "Rancher has been stopped starting K3s." +# Start Cluster without Rancher. +k3s server --cluster-init --disable=traefik,servicelb,metrics-server,local-storage --node-name=local-node --log=./k3s.log >/dev/null 2>&1 & +KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +# Wait for cluster to start. +while ! kubectl version >/dev/null 2>&1; do + echo "Waiting for cluster to start" + sleep 5 +done echo "Uploading new webhook image" +###### Upload the newly created webhook image to containerd, then install the webhook chart using the new image +IMAGE_FILE=./dist/rancher-webhook-image.tar +# import image to containerd and get the image name +WEBHOOK_REPO=$(ctr image import ${IMAGE_FILE} | cut -d ' ' -f 2 | cut -d ':' -f 1) + # Source tags file to get the last built tags source ./dist/tags # Install the webhook chart we just built. -upgrade_rancher_webhook() { - helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system \ - --wait --set image.repository="${IMAGE_REPO}" --set image.tag="${IMAGE_TAG}" --reuse-values --debug -} -try --delay 2 --max 4 --failmsg "Couldn't helm upgrade rancher-webhook" upgrade_rancher_webhook +# This command can fail since it is so close to the cluster start so we will give it 3 retires. +RETRIES=0 +while ! helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tgz -n cattle-system \ + --wait --set image.repository=${WEBHOOK_REPO} --set image.tag=${TAG} --reuse-values; do + if [ "$RETRIES" -ge 3 ]; then + exit 1 + fi + RETRIES=$((RETRIES + 1)) + sleep 2 +done ./bin/rancher-webhook-integration.test -test.v -test.run IntegrationTest @@ -53,5 +92,4 @@ helm upgrade rancher-webhook ./dist/artifacts/rancher-webhook-${HELM_VERSION}.tg # Scale down rancher-webhook so that we can run tests on the FailurePolicy. kubectl scale deploy rancher-webhook -n cattle-system --replicas=0 kubectl wait pods -l app=rancher-webhook --for=delete -n cattle-system - ./bin/rancher-webhook-integration.test -test.v -test.run FailurePolicyTest diff --git a/scripts/package b/scripts/package index 7b330966b..c2d07a3c7 100755 --- a/scripts/package +++ b/scripts/package @@ -3,8 +3,6 @@ set -eu source $(dirname $0)/version -set -x - cd $(dirname $0)/.. echo Running package @@ -32,7 +30,7 @@ fi echo Built ${IMAGE} docker save -o dist/rancher-webhook-image.tar ${IMAGE} -echo TAG="${TAG}" > dist/image_tag +echo IMAGE_TAG="${TAG}" > dist/image_tag if [ "${PUSH:-}" = "true" ]; then docker push ${IMAGE} diff --git a/scripts/package-for-ci b/scripts/package-for-ci deleted file mode 100755 index 368d4efc3..000000000 --- a/scripts/package-for-ci +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -eu - -if [ -z "${SHA:-}" ] ; then - echo "$0 - SHA not set in the environment" - exit 1 -fi - -source $(dirname $0)/version - -set -x - -echo "QQQ: scripts/package: SHA is ${SHA}" -REPO=rancher -IMAGE=${REPO}/webhook:${TAG} -docker tag ${IMAGE} "${REPO}/webhook:${SHA}" -docker save -o dist/rancher-webhook-image-${SHA}.tar "${REPO}/webhook:${SHA}" diff --git a/scripts/package-helm b/scripts/package-helm index 392c32ff1..3f3a94b90 100755 --- a/scripts/package-helm +++ b/scripts/package-helm @@ -12,8 +12,8 @@ cp -rf charts build/ # must use sed -i''` for GNU and OSX compatibility sed -i'.bkp' \ - -e 's/^version:.*/version: '${HELM_VERSION}'/' \ - -e 's/appVersion:.*/appVersion: '${HELM_VERSION}'/' \ + -e 's/^version:.*/version: '${HELM_CHART_VERSION}'/' \ + -e 's/appVersion:.*/appVersion: '${HELM_CHART_VERSION}'/' \ build/charts/rancher-webhook/Chart.yaml sed -i'.bkb' \ diff --git a/scripts/test-helm b/scripts/test-helm index a1e95d1e6..d5d61b696 100755 --- a/scripts/test-helm +++ b/scripts/test-helm @@ -2,7 +2,6 @@ set -e cd $(dirname $0)/.. -./scripts/package-helm echo Running helm lint helm lint ./charts/rancher-webhook # Check for unittest plugin diff --git a/scripts/version b/scripts/version index 5e482d32a..3d25739f4 100755 --- a/scripts/version +++ b/scripts/version @@ -6,7 +6,7 @@ if [ -n "$(git status --porcelain --untracked-files=no)" ]; then fi COMMIT=${COMMIT:-$(git rev-parse --short HEAD)} -GIT_TAG=${GIT_TAG:-${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)}} +GIT_TAG=${GIT_TAG:-$(git tag -l --contains HEAD | head -n 1)} if [[ -z "$DIRTY" && -n "$GIT_TAG" ]]; then VERSION=$GIT_TAG @@ -21,16 +21,16 @@ fi SUFFIX="-${ARCH}" HELM_TAG="${HELM_TAG:-${TAG:-${VERSION}}}" -HELM_VERSION="${HELM_VERSION:-${HELM_TAG/v/}}" +HELM_CHART_VERSION="${HELM_CHART_VERSION:-${HELM_TAG/v/}}" TAG="${TAG:-${VERSION}${SUFFIX}}" REPO="${REPO:-rancher}" if echo $TAG | grep -q dirty; then TAG=dev HELM_TAG=dev - HELM_VERSION=0.0.0-dev + HELM_CHART_VERSION=0.0.0-dev fi DIST_DIR="${DIST_DIR:-$(dirname $0)/../dist/}" mkdir -p ${DIST_DIR} -echo "export TAG=${TAG}; export HELM_TAG=${HELM_TAG}; export HELM_VERSION=${HELM_VERSION};" >${DIST_DIR}/tags +echo "export TAG=${TAG}; export HELM_TAG=${HELM_TAG}; export HELM_CHART_VERSION=${HELM_CHART_VERSION};" >${DIST_DIR}/tags