PWX-37884 Refactor px serviceaccount token integration test

pkg/util/test/util.go

@@ -68,6 +68,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
 	affinityhelper "k8s.io/component-helpers/scheduling/corev1/nodeaffinity"
+	"k8s.io/kubernetes/pkg/apis/core"
 	cluster_v1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/deprecated/v1alpha1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -198,6 +199,8 @@ const (
 
 	etcHostsFile       = "/etc/hosts"
 	tempEtcHostsMarker = "### px-operator unit-test"
+
+	pxSaTokenSecretName = "px-sa-token-secret"
 )
 
 // TestSpecPath is the path for all test specs. Due to currently functional test and
@@ -232,6 +235,7 @@ var (
 	pxVer3_0, _   = version.NewVersion("3.0")
 	pxVer3_1, _   = version.NewVersion("3.1")
 	pxVer3_1_2, _ = version.NewVersion("3.1.2")
+	pxVer3_2, _   = version.NewVersion("3.2")
 
 	// minimumPxVersionCCMJAVA minimum PX version to install ccm-java
 	minimumPxVersionCCMJAVA, _ = version.NewVersion("2.8")
@@ -1031,6 +1035,11 @@ func ValidateStorageCluster(
 		return err
 	}
 
+	// Validate Portworx ServiceAccount Token
+	if err = validatePortworxTokenRefresh(liveCluster, timeout, interval); err != nil {
+		return err
+	}
+
 	// Validate dmthin
 	if err = validateDmthinOnPxNodes(liveCluster); err != nil {
 		return err
@@ -1490,6 +1499,11 @@ func ValidateUninstallStorageCluster(
 		return err
 	}
 
+	// Validate deletion of Px ServiceAccount Token Secret
+	if err := validatePortworxSaTokenSecretDeleted(cluster, timeout, interval); err != nil {
+		return err
+	}
+
 	// Verify telemetry secret is deleted on UninstallAndWipe when telemetry is enabled
 	if cluster.Spec.DeleteStrategy != nil && cluster.Spec.DeleteStrategy.Type == corev1.UninstallAndWipeStorageClusterStrategyType {
 		secret, err := coreops.Instance().GetSecret(TelemetryCertName, cluster.Namespace)
@@ -1545,6 +1559,33 @@ func validatePortworxConfigMapsDeleted(cluster *corev1.StorageCluster, timeout,
 	return nil
 }
 
+func validatePortworxSaTokenSecretDeleted(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
+	pxVersion := GetPortworxVersion(cluster)
+	opVersion, err := GetPxOperatorVersion()
+	if err != nil {
+		return err
+	}
+	if pxVersion.LessThan(pxVer3_2) || opVersion.LessThan(opVer24_2_0) {
+		logrus.Infof("pxVersion: %v, opVersion: %v. Skip verification because px token refresh is not supported with these versions.", pxVersion, opVersion)
+		return nil
+	}
+	t := func() (interface{}, bool, error) {
+		secret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				return nil, false, nil
+			}
+			return nil, true, err
+		}
+		return nil, true, fmt.Errorf("px ServiceAccount Token Secret exists: %v", secret)
+	}
+	if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil {
+		return err
+	}
+	logrus.Debug("Portworx ServiceAccount Token Secret has been deleted successfully")
+	return nil
+}
+
 type podTestFnType func(pod v1.Pod) bool
 
 func validateStorageClusterPods(
@@ -1794,6 +1835,58 @@ func validatePortworxAPIService(cluster *corev1.StorageCluster, timeout, interva
 	return nil
 }
 
+func validatePortworxTokenRefresh(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
+	pxVersion := GetPortworxVersion(cluster)
+	opVersion, err := GetPxOperatorVersion()
+	if err != nil {
+		return err
+	}
+	if pxVersion.LessThan(pxVer3_2) || opVersion.LessThan(opVer24_2_0) {
+		logrus.Infof("pxVersion: %v, opVersion: %v. Skip verification because px token refresh is not supported with these versions.", pxVersion, opVersion)
+		return nil
+	}
+	logrus.Infof("Verifying px runc container token...")
+	// Get one Portworx pod to run commands inside the px runc container on the same node
+	pxPods, err := coreops.Instance().GetPods(cluster.Namespace, map[string]string{"name": "portworx"})
+	if err != nil {
+		return fmt.Errorf("failed to get PX pods, Err: %w", err)
+	}
+	pxPod := pxPods.Items[0]
+	t := func() (interface{}, bool, error) {
+		pxSaSecret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace)
+		if err != nil {
+			return nil, true, err
+		}
+		expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
+		if !coreops.Instance().IsPodReady(pxPod) {
+			return nil, true, fmt.Errorf("[%s] PX pod is not in Ready state to run command inside", pxPod.Name)
+		}
+		actualToken, err := runCmdInsidePxPod(&pxPod, "runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token", cluster.Namespace, false)
+		if err != nil {
+			return nil, true, err
+		}
+		if expectedToken != actualToken {
+			return nil, true, fmt.Errorf("the token inside px runc container is different from the token in the k8s secret")
+		}
+		return actualToken, false, nil
+	}
+	token, err := task.DoRetryWithTimeout(t, timeout, interval)
+	if err != nil {
+		return err
+	}
+	secretList, err := runCmdInsidePxPod(&pxPod, fmt.Sprintf("runc exec portworx "+
+		"curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+
+		"--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt", token), cluster.Namespace, false)
+	if err != nil {
+		return fmt.Errorf("failed to verify px ServiceAccount token: %w", err)
+	}
+	if !strings.Contains(secretList, pxSaTokenSecretName) {
+		return fmt.Errorf("the secret list returned from k8s api server does not contain %s. Output: %s", pxSaTokenSecretName, secretList)
+	}
+	logrus.Infof("token is created and verified: %s", token)
+	return nil
+}
+
 // GetExpectedPxNodeList will get the list of nodes that should be included
 // in the given Portworx cluster, by seeing if each non-master node matches the given
 // node selectors and affinities.

test/integration_test/basic_test.go

@@ -475,45 +475,34 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) {
 		cluster, ok := testSpec.(*corev1.StorageCluster)
 		require.True(t, ok)
 
-		verifyTokenFunc := func() string {
-			pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
-			require.NoError(t, err)
-			expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
-			require.Eventually(t, func() bool {
-				actualToken, stderr, err := ci_utils.RunPxCmd("runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token")
-				require.Empty(t, stderr)
-				require.NoError(t, err)
-				return expectedToken == actualToken
-			}, 10*time.Minute, 15*time.Second, "the token inside px runc container is different from the token in the k8s secret")
-
-			stdout, stderr, err := ci_utils.RunPxCmd(fmt.Sprintf("runc exec portworx "+
-				"curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+
-				"--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | grep %s", expectedToken, pxutil.PortworxServiceAccountTokenSecretName))
-			errMsg := "px not able to communicate with k8s api server with the mounted service account token"
-			require.True(t, strings.Contains(stdout, pxutil.PortworxServiceAccountTokenSecretName),
-				fmt.Sprintf("the secret list returned from k8s api server does not contain %s. output: %s", pxutil.PortworxServiceAccountTokenSecretName, stdout))
-			require.Empty(t, stderr, fmt.Sprintf("%s: %s", errMsg, stderr))
-			require.NoError(t, err, fmt.Sprintf("%s: %s", errMsg, err.Error()))
-			logrus.Infof("token is created and verified: %s", expectedToken)
-			return expectedToken
-		}
-
 		cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t)
+		pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
+		require.NoError(t, err)
+		startupToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
 
-		logrus.Infof("Verifying px container token...")
-		token := verifyTokenFunc()
-
-		time.Sleep(time.Duration(5) * time.Minute)
-		logrus.Infof("Verifying auto-refreshed px runc container token...")
-		refreshedToken := verifyTokenFunc()
-		require.NotEqual(t, token, refreshedToken, "the token did not get refreshed")
+		time.Sleep(5 * time.Minute)
+		require.NoError(t, err)
+		pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
+		require.NoError(t, err)
+		refreshedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
+		require.Eventually(t, func() bool {
+			return startupToken != refreshedToken
+		}, 10*time.Minute, 15*time.Second, "the token did not get refreshed")
+		err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "")
 
-		logrus.Infof("Verifying px runc container token gets recreated after manual deletion...")
-		err := coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
+		err = coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
 		require.NoError(t, err)
-		time.Sleep(time.Duration(2) * time.Minute)
-		recreatedToken := verifyTokenFunc()
-		require.NotEqual(t, refreshedToken, recreatedToken, "the token did not get refreshed")
+		time.Sleep(2 * time.Minute)
+		pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
+		require.NoError(t, err)
+		recreatedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
+		require.Eventually(t, func() bool {
+			return refreshedToken != recreatedToken
+		}, 10*time.Minute, 15*time.Second, "the token did not get refreshed")
+		err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "")
+
+		// Delete and validate the deletion
+		ci_utils.UninstallAndValidateStorageCluster(cluster, t)
 	}
 }