From 7e2266c0bbd1979d53a0c29fe2834215075444c9 Mon Sep 17 00:00:00 2001 From: Shawn Sun <32376495+ssz1997@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:45:14 -0700 Subject: [PATCH] PWX-37884 Refactor px serviceaccount token integration test (#1615) * refactor test Signed-off-by: shsun_pure * uninstall cluster and verify token secret deletion * address comments * fix test --------- Signed-off-by: shsun_pure Co-authored-by: shsun_pure --- pkg/util/test/util.go | 175 +++++++++++++++++++++++++++- test/integration_test/basic_test.go | 49 +++----- 2 files changed, 192 insertions(+), 32 deletions(-) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 073d2498b..8794098c4 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/hashicorp/go-version" consolev1 "github.com/openshift/api/console/v1" routev1 "github.com/openshift/api/route/v1" @@ -55,6 +56,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" affinityhelper "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + "k8s.io/kubernetes/pkg/apis/core" cluster_v1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/deprecated/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -68,7 +70,8 @@ import ( const ( // PxReleaseManifestURLEnvVarName is a release manifest URL Env variable name PxReleaseManifestURLEnvVarName = "PX_RELEASE_MANIFEST_URL" - + // AnnotationPXVersion annotation indicating the portworx semantic version + AnnotationPXVersion = pxAnnotationPrefix + "/px-version" // PxRegistryUserEnvVarName is a Docker username Env variable name PxRegistryUserEnvVarName = "REGISTRY_USER" // PxRegistryPasswordEnvVarName is a Docker password Env variable name @@ -81,14 +84,23 @@ const ( // PxMasterVersion is a tag for Portworx master version PxMasterVersion = "3.0.0.0" + pxAnnotationPrefix = "portworx.io" + etcHostsFile = "/etc/hosts" tempEtcHostsMarker = "### px-operator unit-test" + + pxSaTokenSecretName = "px-sa-token-secret" + + defaultRunCmdInPxPodTimeout = 25 * time.Second + defaultRunCmdInPxPodInterval = 5 * time.Second ) // TestSpecPath is the path for all test specs. Due to currently functional test and // unit test use different path, this needs to be set accordingly. var TestSpecPath = "testspec" +var pxVer3_2, _ = version.NewVersion("3.2") + // MockDriver creates a mock storage driver func MockDriver(mockCtrl *gomock.Controller) *mock.MockDriver { return mock.NewMockDriver(mockCtrl) @@ -484,6 +496,11 @@ func ValidateStorageCluster( return err } + // Validate Portworx ServiceAccount Token + if err = validatePortworxTokenRefresh(liveCluster, timeout, interval); err != nil { + return err + } + return nil } @@ -711,6 +728,35 @@ func ValidateUninstallStorageCluster( if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil { return err } + + // Validate deletion of Px ServiceAccount Token Secret + if err := validatePortworxSaTokenSecretDeleted(cluster, timeout, interval); err != nil { + return err + } + return nil +} + +func validatePortworxSaTokenSecretDeleted(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { + pxVersion := GetPortworxVersion(cluster) + + if pxVersion.LessThan(pxVer3_2) { + logrus.Infof("pxVersion: %v, opVersion: 24.2.0. Skip verification because px token refresh is not supported with these versions.", pxVersion) + return nil + } + t := func() (interface{}, bool, error) { + secret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace) + if err != nil { + if errors.IsNotFound(err) { + return nil, false, nil + } + return nil, true, err + } + return nil, true, fmt.Errorf("px ServiceAccount Token Secret exists: %v", secret) + } + if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil { + return err + } + logrus.Debug("Portworx ServiceAccount Token Secret has been deleted successfully") return nil } @@ -871,7 +917,55 @@ func validatePortworxAPIService(cluster *corev1.StorageCluster, timeout, interva return nil } -// GetExpectedPxNodeNameList will get the list of node names that should be included +func validatePortworxTokenRefresh(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { + pxVersion := GetPortworxVersion(cluster) + if pxVersion.LessThan(pxVer3_2) { + logrus.Infof("pxVersion: %v, opVersion: 24.2.0. Skip verification because px token refresh is not supported with these versions.", pxVersion) + return nil + } + logrus.Infof("Verifying px runc container token...") + // Get one Portworx pod to run commands inside the px runc container on the same node + pxPods, err := coreops.Instance().GetPods(cluster.Namespace, map[string]string{"name": "portworx"}) + if err != nil { + return fmt.Errorf("failed to get PX pods, Err: %w", err) + } + pxPod := pxPods.Items[0] + t := func() (interface{}, bool, error) { + pxSaSecret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace) + if err != nil { + return nil, true, err + } + expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) + if !coreops.Instance().IsPodReady(pxPod) { + return nil, true, fmt.Errorf("[%s] PX pod is not in Ready state to run command inside", pxPod.Name) + } + actualToken, err := runCmdInsidePxPod(&pxPod, "runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token", cluster.Namespace, false) + if err != nil { + return nil, true, err + } + if expectedToken != actualToken { + return nil, true, fmt.Errorf("the token inside px runc container is different from the token in the k8s secret") + } + return actualToken, false, nil + } + token, err := task.DoRetryWithTimeout(t, timeout, interval) + if err != nil { + return err + } + secretList, err := runCmdInsidePxPod(&pxPod, fmt.Sprintf("runc exec portworx "+ + "curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+ + "--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt", token), cluster.Namespace, false) + if err != nil { + return fmt.Errorf("failed to verify px ServiceAccount token: %w", err) + } + if !strings.Contains(secretList, pxSaTokenSecretName) { + return fmt.Errorf("the secret list returned from k8s api server does not contain %s. Output: %s", pxSaTokenSecretName, secretList) + } + logrus.Infof("token is created and verified: %s", token) + return nil +} + +// GetExpectedPxNodeList will get the list of nodes that should be included // in the given Portworx cluster, by seeing if each non-master node matches the given // node selectors and affinities. func GetExpectedPxNodeNameList(cluster *corev1.StorageCluster) ([]string, error) { @@ -2098,6 +2192,83 @@ func validatePodTopologySpreadConstraints(deployment *appsv1.Deployment, timeout return nil } +func runCmdInsidePxPod(pxPod *v1.Pod, cmd string, namespace string, ignoreErr bool) (string, error) { + t := func() (interface{}, bool, error) { + // Execute command in PX pod + cmds := []string{"nsenter", "--mount=/host_proc/1/ns/mnt", "/bin/bash", "-c", cmd} + logrus.Debugf("[%s] Running command inside pod %s", pxPod.Name, cmds) + output, err := coreops.Instance().RunCommandInPod(cmds, pxPod.Name, "portworx", pxPod.Namespace) + if !ignoreErr && err != nil { + return "", true, fmt.Errorf("[%s] failed to run command inside pod, command: %v, err: %v", pxPod.Name, cmds, err) + } + return output, false, err + } + + output, err := task.DoRetryWithTimeout(t, defaultRunCmdInPxPodTimeout, defaultRunCmdInPxPodInterval) + if err != nil { + return "", err + } + + return output.(string), nil +} + +// GetPortworxVersion returns the Portworx version based on the image provided. +// We first look at spec.Image, if not valid image tag found, we check the PX_IMAGE +// env variable. If that is not present or invalid semvar, then we fallback to an +// annotation portworx.io/px-version; then we try to extract the version from PX_RELEASE_MANIFEST_URL +// env variable, else we return master version +func GetPortworxVersion(cluster *corev1.StorageCluster) *version.Version { + var ( + err error + pxVersion *version.Version + ) + + pxImage := cluster.Spec.Image + var manifestURL string + for _, env := range cluster.Spec.Env { + if env.Name == PxImageEnvVarName { + pxImage = env.Value + } else if env.Name == PxReleaseManifestURLEnvVarName { + manifestURL = env.Value + } + } + + pxVersionStr := strings.Split(pxImage, ":")[len(strings.Split(pxImage, ":"))-1] + pxVersion, err = version.NewSemver(pxVersionStr) + if err != nil { + logrus.WithError(err).Warnf("Invalid PX version %s extracted from image name", pxVersionStr) + if pxVersionStr, exists := cluster.Annotations[AnnotationPXVersion]; exists { + logrus.Infof("Checking version in annotations %s", AnnotationPXVersion) + pxVersion, err = version.NewSemver(pxVersionStr) + if err != nil { + logrus.WithError(err).Warnf("Invalid PX version %s extracted from annotation", pxVersionStr) + } + } else { + logrus.Infof("Checking version in %s", PxReleaseManifestURLEnvVarName) + pxVersionStr = getPortworxVersionFromManifestURL(manifestURL) + pxVersion, err = version.NewSemver(pxVersionStr) + if err != nil { + logrus.WithError(err).Warnf("Invalid PX version %s extracted from %s", pxVersionStr, PxReleaseManifestURLEnvVarName) + } + } + } + + if pxVersion == nil { + logrus.Warnf("Failed to determine PX version, assuming its latest and setting it to master: %s", PxMasterVersion) + pxVersion, _ = version.NewVersion(PxMasterVersion) + } + return pxVersion +} + +func getPortworxVersionFromManifestURL(url string) string { + regex := regexp.MustCompile(`.*portworx\.com\/(.*)\/version`) + version := regex.FindStringSubmatch(url) + if len(version) >= 2 { + return version[1] + } + return "" +} + func isPVCControllerEnabled(cluster *corev1.StorageCluster) bool { enabled, err := strconv.ParseBool(cluster.Annotations["portworx.io/pvc-controller"]) if err == nil { diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index a7d48c8d1..e38aa85ee 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -5,7 +5,6 @@ package integrationtest import ( "fmt" - "strings" "testing" "time" @@ -328,45 +327,35 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) { cluster, ok := testSpec.(*corev1.StorageCluster) require.True(t, ok) - verifyTokenFunc := func() string { + verifyTokenRefreshed := func(oldToken string) string { pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) - expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) + newToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) require.Eventually(t, func() bool { - actualToken, stderr, err := ci_utils.RunPxCmd("runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token") - require.Empty(t, stderr) - require.NoError(t, err) - return expectedToken == actualToken - }, 10*time.Minute, 15*time.Second, "the token inside px runc container is different from the token in the k8s secret") - - stdout, stderr, err := ci_utils.RunPxCmd(fmt.Sprintf("runc exec portworx "+ - "curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+ - "--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | grep %s", expectedToken, pxutil.PortworxServiceAccountTokenSecretName)) - errMsg := "px not able to communicate with k8s api server with the mounted service account token" - require.True(t, strings.Contains(stdout, pxutil.PortworxServiceAccountTokenSecretName), - fmt.Sprintf("the secret list returned from k8s api server does not contain %s. output: %s", pxutil.PortworxServiceAccountTokenSecretName, stdout)) - require.Empty(t, stderr, fmt.Sprintf("%s: %s", errMsg, stderr)) - require.NoError(t, err, fmt.Sprintf("%s: %s", errMsg, err.Error())) - logrus.Infof("token is created and verified: %s", expectedToken) - return expectedToken + return oldToken != newToken + }, 10*time.Minute, 15*time.Second, "the token did not get refreshed") + return newToken } cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) + pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + require.NoError(t, err) + startupToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) - logrus.Infof("Verifying px container token...") - token := verifyTokenFunc() + time.Sleep(5 * time.Minute) - time.Sleep(time.Duration(5) * time.Minute) - logrus.Infof("Verifying auto-refreshed px runc container token...") - refreshedToken := verifyTokenFunc() - require.NotEqual(t, token, refreshedToken, "the token did not get refreshed") + refreshedToken := verifyTokenRefreshed(startupToken) + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") - logrus.Infof("Verifying px runc container token gets recreated after manual deletion...") - err := coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + err = coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) - time.Sleep(time.Duration(2) * time.Minute) - recreatedToken := verifyTokenFunc() - require.NotEqual(t, refreshedToken, recreatedToken, "the token did not get refreshed") + time.Sleep(2 * time.Minute) + + verifyTokenRefreshed(refreshedToken) + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") + + // Delete and validate the deletion + ci_utils.UninstallAndValidateStorageCluster(cluster, t) } }