From 4bed599d0b437a0ad2a954a63692b736ec53a1fb Mon Sep 17 00:00:00 2001 From: shsun_pure Date: Thu, 25 Jul 2024 20:12:25 +0000 Subject: [PATCH 1/4] refactor test Signed-off-by: shsun_pure --- pkg/util/test/util.go | 60 +++++++++++++++++++++++++++++ test/integration_test/basic_test.go | 46 +++++++--------------- 2 files changed, 74 insertions(+), 32 deletions(-) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 0458ce7ab..d892ebac5 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -68,6 +68,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" affinityhelper "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + "k8s.io/kubernetes/pkg/apis/core" cluster_v1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/deprecated/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -232,6 +233,7 @@ var ( pxVer3_0, _ = version.NewVersion("3.0") pxVer3_1, _ = version.NewVersion("3.1") pxVer3_1_2, _ = version.NewVersion("3.1.2") + pxVer3_2, _ = version.NewVersion("3.2") // minimumPxVersionCCMJAVA minimum PX version to install ccm-java minimumPxVersionCCMJAVA, _ = version.NewVersion("2.8") @@ -1031,6 +1033,11 @@ func ValidateStorageCluster( return err } + // Validate Portworx ServiceAccount Token + if err = validatePortworxTokenRefresh(liveCluster, timeout, interval); err != nil { + return err + } + // Validate dmthin if err = validateDmthinOnPxNodes(liveCluster); err != nil { return err @@ -1794,6 +1801,59 @@ func validatePortworxAPIService(cluster *corev1.StorageCluster, timeout, interva return nil } +func validatePortworxTokenRefresh(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { + pxVersion := GetPortworxVersion(cluster) + opVersion, err := GetPxOperatorVersion() + if err != nil { + return err + } + if pxVersion.LessThan(pxVer3_2) || opVersion.LessThan(opVer24_2_0) { + logrus.Infof("pxVersion: %v, opVersion: %v. Skip verification because px token refresh is not supported with these versions.", pxVersion, opVersion) + return nil + } + pxSaTokenSecretName := "px-sa-token-secret" + logrus.Infof("Verifying px runc container token...") + // Get one Portworx pod to run commands inside the px runc container on the same node + pxPods, err := coreops.Instance().GetPods(cluster.Namespace, map[string]string{"name": "portworx"}) + if err != nil { + return fmt.Errorf("failed to get PX pods, Err: %w", err) + } + pxPod := pxPods.Items[0] + t := func() (interface{}, bool, error) { + pxSaSecret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace) + if err != nil { + return nil, true, err + } + expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) + if !coreops.Instance().IsPodReady(pxPod) { + return nil, true, fmt.Errorf("[%s] PX pod is not in Ready state to run command inside", pxPod.Name) + } + actualToken, err := runCmdInsidePxPod(&pxPod, "runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token", cluster.Namespace, false) + if err != nil { + return nil, true, err + } + if expectedToken != actualToken { + return nil, true, fmt.Errorf("the token inside px runc container is different from the token in the k8s secret") + } + return actualToken, false, nil + } + token, err := task.DoRetryWithTimeout(t, timeout, interval) + if err != nil { + return err + } + secretList, err := runCmdInsidePxPod(&pxPod, fmt.Sprintf("runc exec portworx "+ + "curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+ + "--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt", token), cluster.Namespace, false) + if err != nil { + return fmt.Errorf("failed to verify px ServiceAccount token: %w", err) + } + if !strings.Contains(secretList, pxSaTokenSecretName) { + return fmt.Errorf("the secret list returned from k8s api server does not contain %s. Output: %s", pxSaTokenSecretName, secretList) + } + logrus.Infof("token is created and verified: %s", token) + return nil +} + // GetExpectedPxNodeList will get the list of nodes that should be included // in the given Portworx cluster, by seeing if each non-master node matches the given // node selectors and affinities. diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index f579a46e0..aafb0967d 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -475,44 +475,26 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) { cluster, ok := testSpec.(*corev1.StorageCluster) require.True(t, ok) - verifyTokenFunc := func() string { - pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) - require.NoError(t, err) - expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) - require.Eventually(t, func() bool { - actualToken, stderr, err := ci_utils.RunPxCmd("runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token") - require.Empty(t, stderr) - require.NoError(t, err) - return expectedToken == actualToken - }, 10*time.Minute, 15*time.Second, "the token inside px runc container is different from the token in the k8s secret") - - stdout, stderr, err := ci_utils.RunPxCmd(fmt.Sprintf("runc exec portworx "+ - "curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+ - "--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | grep %s", expectedToken, pxutil.PortworxServiceAccountTokenSecretName)) - errMsg := "px not able to communicate with k8s api server with the mounted service account token" - require.True(t, strings.Contains(stdout, pxutil.PortworxServiceAccountTokenSecretName), - fmt.Sprintf("the secret list returned from k8s api server does not contain %s. output: %s", pxutil.PortworxServiceAccountTokenSecretName, stdout)) - require.Empty(t, stderr, fmt.Sprintf("%s: %s", errMsg, stderr)) - require.NoError(t, err, fmt.Sprintf("%s: %s", errMsg, err.Error())) - logrus.Infof("token is created and verified: %s", expectedToken) - return expectedToken - } - cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) - - logrus.Infof("Verifying px container token...") - token := verifyTokenFunc() + pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + require.NoError(t, err) + startupToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) time.Sleep(time.Duration(5) * time.Minute) - logrus.Infof("Verifying auto-refreshed px runc container token...") - refreshedToken := verifyTokenFunc() - require.NotEqual(t, token, refreshedToken, "the token did not get refreshed") + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") + require.NoError(t, err) + pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + require.NoError(t, err) + refreshedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) + require.NotEqual(t, startupToken, refreshedToken, "the token did not get refreshed") - logrus.Infof("Verifying px runc container token gets recreated after manual deletion...") - err := coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + err = coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) time.Sleep(time.Duration(2) * time.Minute) - recreatedToken := verifyTokenFunc() + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") + pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + require.NoError(t, err) + recreatedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) require.NotEqual(t, refreshedToken, recreatedToken, "the token did not get refreshed") } } From 131d209999d1311153d4fc6fe1b8a30c0445a9e1 Mon Sep 17 00:00:00 2001 From: shsun_pure Date: Fri, 26 Jul 2024 16:39:33 +0000 Subject: [PATCH 2/4] uninstall cluster and verify token secret deletion --- pkg/util/test/util.go | 35 ++++++++++++++++++++++++++++- test/integration_test/basic_test.go | 3 +++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index d892ebac5..5b0a7fefa 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -199,6 +199,8 @@ const ( etcHostsFile = "/etc/hosts" tempEtcHostsMarker = "### px-operator unit-test" + + pxSaTokenSecretName = "px-sa-token-secret" ) // TestSpecPath is the path for all test specs. Due to currently functional test and @@ -1497,6 +1499,11 @@ func ValidateUninstallStorageCluster( return err } + // Validate deletion of Px ServiceAccount Token Secret + if err := validatePortworxSaTokenSecretDeleted(cluster, timeout, interval); err != nil { + return err + } + // Verify telemetry secret is deleted on UninstallAndWipe when telemetry is enabled if cluster.Spec.DeleteStrategy != nil && cluster.Spec.DeleteStrategy.Type == corev1.UninstallAndWipeStorageClusterStrategyType { secret, err := coreops.Instance().GetSecret(TelemetryCertName, cluster.Namespace) @@ -1552,6 +1559,33 @@ func validatePortworxConfigMapsDeleted(cluster *corev1.StorageCluster, timeout, return nil } +func validatePortworxSaTokenSecretDeleted(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { + pxVersion := GetPortworxVersion(cluster) + opVersion, err := GetPxOperatorVersion() + if err != nil { + return err + } + if pxVersion.LessThan(pxVer3_2) || opVersion.LessThan(opVer24_2_0) { + logrus.Infof("pxVersion: %v, opVersion: %v. Skip verification because px token refresh is not supported with these versions.", pxVersion, opVersion) + return nil + } + t := func() (interface{}, bool, error) { + secret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace) + if err != nil { + if errors.IsNotFound(err) { + return nil, false, nil + } + return nil, true, err + } + return nil, true, fmt.Errorf("px ServiceAccount Token Secret exists: %v", secret) + } + if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil { + return err + } + logrus.Debug("Portworx ServiceAccount Token Secret has been deleted successfully") + return nil +} + type podTestFnType func(pod v1.Pod) bool func validateStorageClusterPods( @@ -1811,7 +1845,6 @@ func validatePortworxTokenRefresh(cluster *corev1.StorageCluster, timeout, inter logrus.Infof("pxVersion: %v, opVersion: %v. Skip verification because px token refresh is not supported with these versions.", pxVersion, opVersion) return nil } - pxSaTokenSecretName := "px-sa-token-secret" logrus.Infof("Verifying px runc container token...") // Get one Portworx pod to run commands inside the px runc container on the same node pxPods, err := coreops.Instance().GetPods(cluster.Namespace, map[string]string{"name": "portworx"}) diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index aafb0967d..698522fb7 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -496,6 +496,9 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) { require.NoError(t, err) recreatedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) require.NotEqual(t, refreshedToken, recreatedToken, "the token did not get refreshed") + + // Delete and validate the deletion + ci_utils.UninstallAndValidateStorageCluster(cluster, t) } } From 738672253ec0155b71487eb56a0f45f647723c4e Mon Sep 17 00:00:00 2001 From: shsun_pure Date: Fri, 26 Jul 2024 16:50:24 +0000 Subject: [PATCH 3/4] address comments --- test/integration_test/basic_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index 698522fb7..234600b3f 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -480,22 +480,26 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) { require.NoError(t, err) startupToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) - time.Sleep(time.Duration(5) * time.Minute) - err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") + time.Sleep(5 * time.Minute) require.NoError(t, err) pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) refreshedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) - require.NotEqual(t, startupToken, refreshedToken, "the token did not get refreshed") + require.Eventually(t, func() bool { + return startupToken != refreshedToken + }, 10*time.Minute, 15*time.Second, "the token did not get refreshed") + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") err = coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) - time.Sleep(time.Duration(2) * time.Minute) - err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") + time.Sleep(2 * time.Minute) pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) recreatedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) - require.NotEqual(t, refreshedToken, recreatedToken, "the token did not get refreshed") + require.Eventually(t, func() bool { + return refreshedToken != recreatedToken + }, 10*time.Minute, 15*time.Second, "the token did not get refreshed") + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") // Delete and validate the deletion ci_utils.UninstallAndValidateStorageCluster(cluster, t) From 06c8e463504e914a374efe236bb9549a5e3f7add Mon Sep 17 00:00:00 2001 From: shsun_pure Date: Mon, 29 Jul 2024 19:48:07 +0000 Subject: [PATCH 4/4] fix test --- test/integration_test/basic_test.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index 234600b3f..f5620e698 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -475,30 +475,31 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) { cluster, ok := testSpec.(*corev1.StorageCluster) require.True(t, ok) + verifyTokenRefreshed := func(oldToken string) string { + pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + require.NoError(t, err) + newToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) + require.Eventually(t, func() bool { + return oldToken != newToken + }, 10*time.Minute, 15*time.Second, "the token did not get refreshed") + return newToken + } + cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) startupToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) time.Sleep(5 * time.Minute) - require.NoError(t, err) - pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) - require.NoError(t, err) - refreshedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) - require.Eventually(t, func() bool { - return startupToken != refreshedToken - }, 10*time.Minute, 15*time.Second, "the token did not get refreshed") + + refreshedToken := verifyTokenRefreshed(startupToken) err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") err = coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) time.Sleep(2 * time.Minute) - pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) - require.NoError(t, err) - recreatedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) - require.Eventually(t, func() bool { - return refreshedToken != recreatedToken - }, 10*time.Minute, 15*time.Second, "the token did not get refreshed") + + verifyTokenRefreshed(refreshedToken) err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") // Delete and validate the deletion