PWX-37884 Refactor px serviceaccount token integration test (#1615)

* refactor test Signed-off-by: shsun_pure <shsun@purestorage.com> * uninstall cluster and verify token secret deletion * address comments * fix test --------- Signed-off-by: shsun_pure <shsun@purestorage.com> Co-authored-by: shsun_pure <shsun@purestorage.com>
libopenstorage · Jul 29, 2024 · 7e2266c · 7e2266c
1 parent d7b955a
commit 7e2266c
Show file tree

Hide file tree

Showing 2 changed files with 192 additions and 32 deletions.
diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/go-version"
 	consolev1 "github.com/openshift/api/console/v1"
 	routev1 "github.com/openshift/api/route/v1"
 
@@ -55,6 +56,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
 	affinityhelper "k8s.io/component-helpers/scheduling/corev1/nodeaffinity"
+	"k8s.io/kubernetes/pkg/apis/core"
 	cluster_v1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/deprecated/v1alpha1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -68,7 +70,8 @@ import (
 const (
 	// PxReleaseManifestURLEnvVarName is a release manifest URL Env variable name
 	PxReleaseManifestURLEnvVarName = "PX_RELEASE_MANIFEST_URL"
-
+	// AnnotationPXVersion annotation indicating the portworx semantic version
+	AnnotationPXVersion = pxAnnotationPrefix + "/px-version"
 	// PxRegistryUserEnvVarName is a Docker username Env variable name
 	PxRegistryUserEnvVarName = "REGISTRY_USER"
 	// PxRegistryPasswordEnvVarName is a Docker password Env variable name
@@ -81,14 +84,23 @@ const (
 	// PxMasterVersion is a tag for Portworx master version
 	PxMasterVersion = "3.0.0.0"
 
+	pxAnnotationPrefix = "portworx.io"
+
 	etcHostsFile       = "/etc/hosts"
 	tempEtcHostsMarker = "### px-operator unit-test"
+
+	pxSaTokenSecretName = "px-sa-token-secret"
+
+	defaultRunCmdInPxPodTimeout  = 25 * time.Second
+	defaultRunCmdInPxPodInterval = 5 * time.Second
 )
 
 // TestSpecPath is the path for all test specs. Due to currently functional test and
 // unit test use different path, this needs to be set accordingly.
 var TestSpecPath = "testspec"
 
+var pxVer3_2, _ = version.NewVersion("3.2")
+
 // MockDriver creates a mock storage driver
 func MockDriver(mockCtrl *gomock.Controller) *mock.MockDriver {
 	return mock.NewMockDriver(mockCtrl)
@@ -484,6 +496,11 @@ func ValidateStorageCluster(
 		return err
 	}
 
+	// Validate Portworx ServiceAccount Token
+	if err = validatePortworxTokenRefresh(liveCluster, timeout, interval); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -711,6 +728,35 @@ func ValidateUninstallStorageCluster(
 	if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil {
 		return err
 	}
+
+	// Validate deletion of Px ServiceAccount Token Secret
+	if err := validatePortworxSaTokenSecretDeleted(cluster, timeout, interval); err != nil {
+		return err
+	}
+	return nil
+}
+
+func validatePortworxSaTokenSecretDeleted(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
+	pxVersion := GetPortworxVersion(cluster)
+
+	if pxVersion.LessThan(pxVer3_2) {
+		logrus.Infof("pxVersion: %v, opVersion: 24.2.0. Skip verification because px token refresh is not supported with these versions.", pxVersion)
+		return nil
+	}
+	t := func() (interface{}, bool, error) {
+		secret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				return nil, false, nil
+			}
+			return nil, true, err
+		}
+		return nil, true, fmt.Errorf("px ServiceAccount Token Secret exists: %v", secret)
+	}
+	if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil {
+		return err
+	}
+	logrus.Debug("Portworx ServiceAccount Token Secret has been deleted successfully")
 	return nil
 }
 
@@ -871,7 +917,55 @@ func validatePortworxAPIService(cluster *corev1.StorageCluster, timeout, interva
 	return nil
 }
 
-// GetExpectedPxNodeNameList will get the list of node names that should be included
+func validatePortworxTokenRefresh(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
+	pxVersion := GetPortworxVersion(cluster)
+	if pxVersion.LessThan(pxVer3_2) {
+		logrus.Infof("pxVersion: %v, opVersion: 24.2.0. Skip verification because px token refresh is not supported with these versions.", pxVersion)
+		return nil
+	}
+	logrus.Infof("Verifying px runc container token...")
+	// Get one Portworx pod to run commands inside the px runc container on the same node
+	pxPods, err := coreops.Instance().GetPods(cluster.Namespace, map[string]string{"name": "portworx"})
+	if err != nil {
+		return fmt.Errorf("failed to get PX pods, Err: %w", err)
+	}
+	pxPod := pxPods.Items[0]
+	t := func() (interface{}, bool, error) {
+		pxSaSecret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace)
+		if err != nil {
+			return nil, true, err
+		}
+		expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
+		if !coreops.Instance().IsPodReady(pxPod) {
+			return nil, true, fmt.Errorf("[%s] PX pod is not in Ready state to run command inside", pxPod.Name)
+		}
+		actualToken, err := runCmdInsidePxPod(&pxPod, "runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token", cluster.Namespace, false)
+		if err != nil {
+			return nil, true, err
+		}
+		if expectedToken != actualToken {
+			return nil, true, fmt.Errorf("the token inside px runc container is different from the token in the k8s secret")
+		}
+		return actualToken, false, nil
+	}
+	token, err := task.DoRetryWithTimeout(t, timeout, interval)
+	if err != nil {
+		return err
+	}
+	secretList, err := runCmdInsidePxPod(&pxPod, fmt.Sprintf("runc exec portworx "+
+		"curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+
+		"--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt", token), cluster.Namespace, false)
+	if err != nil {
+		return fmt.Errorf("failed to verify px ServiceAccount token: %w", err)
+	}
+	if !strings.Contains(secretList, pxSaTokenSecretName) {
+		return fmt.Errorf("the secret list returned from k8s api server does not contain %s. Output: %s", pxSaTokenSecretName, secretList)
+	}
+	logrus.Infof("token is created and verified: %s", token)
+	return nil
+}
+
+// GetExpectedPxNodeList will get the list of nodes that should be included
 // in the given Portworx cluster, by seeing if each non-master node matches the given
 // node selectors and affinities.
 func GetExpectedPxNodeNameList(cluster *corev1.StorageCluster) ([]string, error) {
@@ -2098,6 +2192,83 @@ func validatePodTopologySpreadConstraints(deployment *appsv1.Deployment, timeout
 	return nil
 }
 
+func runCmdInsidePxPod(pxPod *v1.Pod, cmd string, namespace string, ignoreErr bool) (string, error) {
+	t := func() (interface{}, bool, error) {
+		// Execute command in PX pod
+		cmds := []string{"nsenter", "--mount=/host_proc/1/ns/mnt", "/bin/bash", "-c", cmd}
+		logrus.Debugf("[%s] Running command inside pod %s", pxPod.Name, cmds)
+		output, err := coreops.Instance().RunCommandInPod(cmds, pxPod.Name, "portworx", pxPod.Namespace)
+		if !ignoreErr && err != nil {
+			return "", true, fmt.Errorf("[%s] failed to run command inside pod, command: %v, err: %v", pxPod.Name, cmds, err)
+		}
+		return output, false, err
+	}
+
+	output, err := task.DoRetryWithTimeout(t, defaultRunCmdInPxPodTimeout, defaultRunCmdInPxPodInterval)
+	if err != nil {
+		return "", err
+	}
+
+	return output.(string), nil
+}
+
+// GetPortworxVersion returns the Portworx version based on the image provided.
+// We first look at spec.Image, if not valid image tag found, we check the PX_IMAGE
+// env variable. If that is not present or invalid semvar, then we fallback to an
+// annotation portworx.io/px-version; then we try to extract the version from PX_RELEASE_MANIFEST_URL
+// env variable, else we return master version
+func GetPortworxVersion(cluster *corev1.StorageCluster) *version.Version {
+	var (
+		err       error
+		pxVersion *version.Version
+	)
+
+	pxImage := cluster.Spec.Image
+	var manifestURL string
+	for _, env := range cluster.Spec.Env {
+		if env.Name == PxImageEnvVarName {
+			pxImage = env.Value
+		} else if env.Name == PxReleaseManifestURLEnvVarName {
+			manifestURL = env.Value
+		}
+	}
+
+	pxVersionStr := strings.Split(pxImage, ":")[len(strings.Split(pxImage, ":"))-1]
+	pxVersion, err = version.NewSemver(pxVersionStr)
+	if err != nil {
+		logrus.WithError(err).Warnf("Invalid PX version %s extracted from image name", pxVersionStr)
+		if pxVersionStr, exists := cluster.Annotations[AnnotationPXVersion]; exists {
+			logrus.Infof("Checking version in annotations %s", AnnotationPXVersion)
+			pxVersion, err = version.NewSemver(pxVersionStr)
+			if err != nil {
+				logrus.WithError(err).Warnf("Invalid PX version %s extracted from annotation", pxVersionStr)
+			}
+		} else {
+			logrus.Infof("Checking version in %s", PxReleaseManifestURLEnvVarName)
+			pxVersionStr = getPortworxVersionFromManifestURL(manifestURL)
+			pxVersion, err = version.NewSemver(pxVersionStr)
+			if err != nil {
+				logrus.WithError(err).Warnf("Invalid PX version %s extracted from %s", pxVersionStr, PxReleaseManifestURLEnvVarName)
+			}
+		}
+	}
+
+	if pxVersion == nil {
+		logrus.Warnf("Failed to determine PX version, assuming its latest and setting it to master: %s", PxMasterVersion)
+		pxVersion, _ = version.NewVersion(PxMasterVersion)
+	}
+	return pxVersion
+}
+
+func getPortworxVersionFromManifestURL(url string) string {
+	regex := regexp.MustCompile(`.*portworx\.com\/(.*)\/version`)
+	version := regex.FindStringSubmatch(url)
+	if len(version) >= 2 {
+		return version[1]
+	}
+	return ""
+}
+
 func isPVCControllerEnabled(cluster *corev1.StorageCluster) bool {
 	enabled, err := strconv.ParseBool(cluster.Annotations["portworx.io/pvc-controller"])
 	if err == nil {

diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go
@@ -5,7 +5,6 @@ package integrationtest
 
 import (
 	"fmt"
-	"strings"
 	"testing"
 	"time"
 
@@ -328,45 +327,35 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) {
 		cluster, ok := testSpec.(*corev1.StorageCluster)
 		require.True(t, ok)
 
-		verifyTokenFunc := func() string {
+		verifyTokenRefreshed := func(oldToken string) string {
 			pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
 			require.NoError(t, err)
-			expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
+			newToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
 			require.Eventually(t, func() bool {
-				actualToken, stderr, err := ci_utils.RunPxCmd("runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token")
-				require.Empty(t, stderr)
-				require.NoError(t, err)
-				return expectedToken == actualToken
-			}, 10*time.Minute, 15*time.Second, "the token inside px runc container is different from the token in the k8s secret")
-
-			stdout, stderr, err := ci_utils.RunPxCmd(fmt.Sprintf("runc exec portworx "+
-				"curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+
-				"--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | grep %s", expectedToken, pxutil.PortworxServiceAccountTokenSecretName))
-			errMsg := "px not able to communicate with k8s api server with the mounted service account token"
-			require.True(t, strings.Contains(stdout, pxutil.PortworxServiceAccountTokenSecretName),
-				fmt.Sprintf("the secret list returned from k8s api server does not contain %s. output: %s", pxutil.PortworxServiceAccountTokenSecretName, stdout))
-			require.Empty(t, stderr, fmt.Sprintf("%s: %s", errMsg, stderr))
-			require.NoError(t, err, fmt.Sprintf("%s: %s", errMsg, err.Error()))
-			logrus.Infof("token is created and verified: %s", expectedToken)
-			return expectedToken
+				return oldToken != newToken
+			}, 10*time.Minute, 15*time.Second, "the token did not get refreshed")
+			return newToken
 		}
 
 		cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t)
+		pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
+		require.NoError(t, err)
+		startupToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
 
-		logrus.Infof("Verifying px container token...")
-		token := verifyTokenFunc()
+		time.Sleep(5 * time.Minute)
 
-		time.Sleep(time.Duration(5) * time.Minute)
-		logrus.Infof("Verifying auto-refreshed px runc container token...")
-		refreshedToken := verifyTokenFunc()
-		require.NotEqual(t, token, refreshedToken, "the token did not get refreshed")
+		refreshedToken := verifyTokenRefreshed(startupToken)
+		err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "")
 
-		logrus.Infof("Verifying px runc container token gets recreated after manual deletion...")
-		err := coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
+		err = coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
 		require.NoError(t, err)
-		time.Sleep(time.Duration(2) * time.Minute)
-		recreatedToken := verifyTokenFunc()
-		require.NotEqual(t, refreshedToken, recreatedToken, "the token did not get refreshed")
+		time.Sleep(2 * time.Minute)
+
+		verifyTokenRefreshed(refreshedToken)
+		err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "")
+
+		// Delete and validate the deletion
+		ci_utils.UninstallAndValidateStorageCluster(cluster, t)
 	}
 }