From 21250faf29ca4a7d31fdc709fa6363d12621fcd1 Mon Sep 17 00:00:00 2001 From: Jashandeep Sohi Date: Mon, 1 Apr 2024 16:47:09 -0700 Subject: [PATCH] test: fix e2e test kind setup --- .github/workflows/e2e.yaml | 35 ++++++++++------- tests/e2e/kind_provider_test.go | 67 +++++++++++++++++++++++++++++++++ tests/e2e/main_test.go | 38 ++++++++++++++++--- 3 files changed, 122 insertions(+), 18 deletions(-) create mode 100644 tests/e2e/kind_provider_test.go diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 797899ca..66d676de 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -11,7 +11,7 @@ defaults: shell: bash env: - KIND_VERSION: v0.19.0 + KIND_VERSION: v0.22.0 OPERATOR_IMAGE_PATH: /tmp/temporal-operator.tar WORKER_PROCESS_IMAGE_PATH: /tmp/example-worker-process.tar @@ -21,12 +21,17 @@ jobs: strategy: fail-fast: false matrix: - kube-version: - - v1.25.16 - - v1.26.14 - - v1.27.11 - - v1.28.7 - - v1.29.2 + kube-version: + - tag: v1.25.16 + kind-image: kindest/node:v1.25.16@sha256:e8b50f8e06b44bb65a93678a65a26248fae585b3d3c2a669e5ca6c90c69dc519 + - tag: v1.26.14 + kind-image: kindest/node:v1.26.14@sha256:5d548739ddef37b9318c70cb977f57bf3e5015e4552be4e27e57280a8cbb8e4f + - tag: v1.27.11 + kind-image: kindest/node:v1.27.11@sha256:681253009e68069b8e01aad36a1e0fa8cf18bb0ab3e5c4069b2e65cafdd70843 + - tag: v1.28.7 + kind-image: kindest/node:v1.28.7@sha256:9bc6c451a289cf96ad0bbaf33d416901de6fd632415b076ab05f5fa7e4f65c58 + - tag: v1.29.2 + kind-image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245 name: Run generate E2E tests steps: - name: Checkout @@ -57,23 +62,27 @@ jobs: cache-from: type=gha cache-to: type=gha outputs: type=docker,dest=${{ env.WORKER_PROCESS_IMAGE_PATH }} - # e2e-framework uses kind v0.12.0 as default value - - name: Install kind - uses: helm/kind-action@v1.9.0 + + - name: Install Kind + uses: helm/kind-action@v1 with: - version: ${{ env.KIND_VERSION }} install_only: true + version: ${{ env.KIND_VERSION }} + - name: Free some disk space run: sudo rm -rf /usr/share/dotnet && sudo rm -rf /opt/ghc && sudo rm -rf "/usr/local/share/boost" + - name: Run e2e test suite run: make test-e2e env: OPERATOR_IMAGE_PATH: ${{ env.OPERATOR_IMAGE_PATH }} WORKER_PROCESS_IMAGE_PATH: ${{ env.WORKER_PROCESS_IMAGE_PATH }} - KUBERNETES_VERSION: ${{ matrix.kube-version }} + KUBERNETES_VERSION: ${{ matrix.kube-version.tag }} + KIND_IMAGE: ${{ matrix.kube-version.kind-image }} + - name: Archive e2e logs uses: actions/upload-artifact@v3 if: always() with: name: e2e-artifacts - path: out/tests/e2e/ \ No newline at end of file + path: out/tests/e2e/ diff --git a/tests/e2e/kind_provider_test.go b/tests/e2e/kind_provider_test.go new file mode 100644 index 00000000..59a26cb5 --- /dev/null +++ b/tests/e2e/kind_provider_test.go @@ -0,0 +1,67 @@ +// Licensed to Alexandre VILAIN under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Alexandre VILAIN licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package e2e + +import ( + "context" + + "sigs.k8s.io/e2e-framework/support" + "sigs.k8s.io/e2e-framework/support/kind" +) + +// FixedKindProvider patches support/kind.Cluster so that it honors the WithImage option. +// TODO: remove when https://github.com/kubernetes-sigs/e2e-framework/pull/395 or similar is available in a e2e-framework release. +type FixedKindProvider struct { + *kind.Cluster + image string +} + +func (k *FixedKindProvider) SetDefaults() support.E2EClusterProvider { + k.Cluster.SetDefaults() + return k +} + +func (k *FixedKindProvider) WithName(name string) support.E2EClusterProvider { + k.Cluster.WithName(name) + return k +} + +func (k *FixedKindProvider) WithVersion(version string) support.E2EClusterProvider { + k.Cluster.WithVersion(version) + return k +} + +func (k *FixedKindProvider) WithPath(path string) support.E2EClusterProvider { + k.Cluster.WithPath(path) + return k +} + +func (k *FixedKindProvider) WithOpts(opts ...support.ClusterOpts) support.E2EClusterProvider { + k.Cluster.WithOpts(opts...) + return k +} + +// Ensure interface is implemented. +var _ support.E2EClusterProvider = &FixedKindProvider{} + +func (k *FixedKindProvider) Create(ctx context.Context, args ...string) (string, error) { + if k.image != "" { + args = append(args, "--image", k.image) + } + return k.Cluster.Create(ctx, args...) +} diff --git a/tests/e2e/main_test.go b/tests/e2e/main_test.go index 5a101208..13b63da3 100644 --- a/tests/e2e/main_test.go +++ b/tests/e2e/main_test.go @@ -30,6 +30,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" "k8s.io/klog/v2/textlogger" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/e2e-framework/klient/decoder" @@ -70,11 +71,11 @@ func testMainRun(m *testing.M) int { clusterLogsOutPath := path.Join(wd, "..", "..", "out", "tests", "e2e") + // if set, enforce API server reported version matches this kubernetesVersion := os.Getenv("KUBERNETES_VERSION") - if kubernetesVersion == "" { - kubernetesVersion = "v1.26.0" - } - kindImage := fmt.Sprintf("kindest/node:%s", kubernetesVersion) + + // if not set default to Kind's default image + kindImage := os.Getenv("KIND_IMAGE") operatorImagePath := os.Getenv("OPERATOR_IMAGE_PATH") exampleWorkerProcessImagePath := os.Getenv("WORKER_PROCESS_IMAGE_PATH") @@ -92,7 +93,10 @@ func testMainRun(m *testing.M) int { return err } - kindCluster := kind.NewProvider().WithOpts(kind.WithImage(kindImage)) + kindCluster := &FixedKindProvider{ + Cluster: &kind.Cluster{}, + image: kindImage, + } testenv = env. NewWithConfig(cfg). @@ -103,6 +107,30 @@ func testMainRun(m *testing.M) int { envfuncs.LoadImageArchiveToCluster(kindClusterName, exampleWorkerProcessImagePath), envfuncs.SetupCRDs("../../out/release/artifacts", "*.crds.yaml"), ). + // Make sure the cluster version is what we expect + Setup(func(ctx context.Context, c *envconf.Config) (context.Context, error) { + if kubernetesVersion == "" { + return ctx, nil + } + + dc, err := discovery.NewDiscoveryClientForConfig(c.Client().RESTConfig()) + + if err != nil { + return ctx, err + } + + sv, err := dc.ServerVersion() + + if err != nil { + return ctx, err + } + + if sv.GitVersion != kubernetesVersion { + return ctx, fmt.Errorf("API server version %v does not match expected value %v", sv.GitVersion, kubernetesVersion) + } + + return ctx, nil + }). // Add the operators crds to the client scheme. Setup(func(ctx context.Context, c *envconf.Config) (context.Context, error) { fmt.Printf("KUBECONFIG=%s\n", c.KubeconfigFile())