diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 0458ce7ab..d892ebac5 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -68,6 +68,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" affinityhelper "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" + "k8s.io/kubernetes/pkg/apis/core" cluster_v1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/deprecated/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -232,6 +233,7 @@ var ( pxVer3_0, _ = version.NewVersion("3.0") pxVer3_1, _ = version.NewVersion("3.1") pxVer3_1_2, _ = version.NewVersion("3.1.2") + pxVer3_2, _ = version.NewVersion("3.2") // minimumPxVersionCCMJAVA minimum PX version to install ccm-java minimumPxVersionCCMJAVA, _ = version.NewVersion("2.8") @@ -1031,6 +1033,11 @@ func ValidateStorageCluster( return err } + // Validate Portworx ServiceAccount Token + if err = validatePortworxTokenRefresh(liveCluster, timeout, interval); err != nil { + return err + } + // Validate dmthin if err = validateDmthinOnPxNodes(liveCluster); err != nil { return err @@ -1794,6 +1801,59 @@ func validatePortworxAPIService(cluster *corev1.StorageCluster, timeout, interva return nil } +func validatePortworxTokenRefresh(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { + pxVersion := GetPortworxVersion(cluster) + opVersion, err := GetPxOperatorVersion() + if err != nil { + return err + } + if pxVersion.LessThan(pxVer3_2) || opVersion.LessThan(opVer24_2_0) { + logrus.Infof("pxVersion: %v, opVersion: %v. Skip verification because px token refresh is not supported with these versions.", pxVersion, opVersion) + return nil + } + pxSaTokenSecretName := "px-sa-token-secret" + logrus.Infof("Verifying px runc container token...") + // Get one Portworx pod to run commands inside the px runc container on the same node + pxPods, err := coreops.Instance().GetPods(cluster.Namespace, map[string]string{"name": "portworx"}) + if err != nil { + return fmt.Errorf("failed to get PX pods, Err: %w", err) + } + pxPod := pxPods.Items[0] + t := func() (interface{}, bool, error) { + pxSaSecret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace) + if err != nil { + return nil, true, err + } + expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) + if !coreops.Instance().IsPodReady(pxPod) { + return nil, true, fmt.Errorf("[%s] PX pod is not in Ready state to run command inside", pxPod.Name) + } + actualToken, err := runCmdInsidePxPod(&pxPod, "runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token", cluster.Namespace, false) + if err != nil { + return nil, true, err + } + if expectedToken != actualToken { + return nil, true, fmt.Errorf("the token inside px runc container is different from the token in the k8s secret") + } + return actualToken, false, nil + } + token, err := task.DoRetryWithTimeout(t, timeout, interval) + if err != nil { + return err + } + secretList, err := runCmdInsidePxPod(&pxPod, fmt.Sprintf("runc exec portworx "+ + "curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+ + "--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt", token), cluster.Namespace, false) + if err != nil { + return fmt.Errorf("failed to verify px ServiceAccount token: %w", err) + } + if !strings.Contains(secretList, pxSaTokenSecretName) { + return fmt.Errorf("the secret list returned from k8s api server does not contain %s. Output: %s", pxSaTokenSecretName, secretList) + } + logrus.Infof("token is created and verified: %s", token) + return nil +} + // GetExpectedPxNodeList will get the list of nodes that should be included // in the given Portworx cluster, by seeing if each non-master node matches the given // node selectors and affinities. diff --git a/test/integration_test/basic_test.go b/test/integration_test/basic_test.go index f579a46e0..aafb0967d 100644 --- a/test/integration_test/basic_test.go +++ b/test/integration_test/basic_test.go @@ -475,44 +475,26 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) { cluster, ok := testSpec.(*corev1.StorageCluster) require.True(t, ok) - verifyTokenFunc := func() string { - pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) - require.NoError(t, err) - expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) - require.Eventually(t, func() bool { - actualToken, stderr, err := ci_utils.RunPxCmd("runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token") - require.Empty(t, stderr) - require.NoError(t, err) - return expectedToken == actualToken - }, 10*time.Minute, 15*time.Second, "the token inside px runc container is different from the token in the k8s secret") - - stdout, stderr, err := ci_utils.RunPxCmd(fmt.Sprintf("runc exec portworx "+ - "curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+ - "--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | grep %s", expectedToken, pxutil.PortworxServiceAccountTokenSecretName)) - errMsg := "px not able to communicate with k8s api server with the mounted service account token" - require.True(t, strings.Contains(stdout, pxutil.PortworxServiceAccountTokenSecretName), - fmt.Sprintf("the secret list returned from k8s api server does not contain %s. output: %s", pxutil.PortworxServiceAccountTokenSecretName, stdout)) - require.Empty(t, stderr, fmt.Sprintf("%s: %s", errMsg, stderr)) - require.NoError(t, err, fmt.Sprintf("%s: %s", errMsg, err.Error())) - logrus.Infof("token is created and verified: %s", expectedToken) - return expectedToken - } - cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) - - logrus.Infof("Verifying px container token...") - token := verifyTokenFunc() + pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + require.NoError(t, err) + startupToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) time.Sleep(time.Duration(5) * time.Minute) - logrus.Infof("Verifying auto-refreshed px runc container token...") - refreshedToken := verifyTokenFunc() - require.NotEqual(t, token, refreshedToken, "the token did not get refreshed") + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") + require.NoError(t, err) + pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + require.NoError(t, err) + refreshedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) + require.NotEqual(t, startupToken, refreshedToken, "the token did not get refreshed") - logrus.Infof("Verifying px runc container token gets recreated after manual deletion...") - err := coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + err = coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) require.NoError(t, err) time.Sleep(time.Duration(2) * time.Minute) - recreatedToken := verifyTokenFunc() + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "") + pxSaSecret, err = coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace) + require.NoError(t, err) + recreatedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey]) require.NotEqual(t, refreshedToken, recreatedToken, "the token did not get refreshed") } }