Skip to content

Commit

Permalink
PWX-37884 Refactor px serviceaccount token integration test (#1615)
Browse files Browse the repository at this point in the history 
* refactor test

Signed-off-by: shsun_pure <shsun@purestorage.com>

* uninstall cluster and verify token secret deletion

* address comments

* fix test

---------

Signed-off-by: shsun_pure <shsun@purestorage.com>
Co-authored-by: shsun_pure <shsun@purestorage.com>
  • Loading branch information
@ssz1997
ssz1997 and shsun_pure authored Jul 29, 2024
1 parent a6ffcfc commit 28cc6a7
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 29 deletions.
93 changes: 93 additions & 0 deletions pkg/util/test/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes/scheme"
affinityhelper "k8s.io/component-helpers/scheduling/corev1/nodeaffinity"
"k8s.io/kubernetes/pkg/apis/core"
cluster_v1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/deprecated/v1alpha1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
Expand Down Expand Up @@ -198,6 +199,8 @@ const (

etcHostsFile = "/etc/hosts"
tempEtcHostsMarker = "### px-operator unit-test"

pxSaTokenSecretName = "px-sa-token-secret"
)

// TestSpecPath is the path for all test specs. Due to currently functional test and
Expand Down Expand Up @@ -232,6 +235,7 @@ var (
pxVer3_0, _ = version.NewVersion("3.0")
pxVer3_1, _ = version.NewVersion("3.1")
pxVer3_1_2, _ = version.NewVersion("3.1.2")
pxVer3_2, _ = version.NewVersion("3.2")

// minimumPxVersionCCMJAVA minimum PX version to install ccm-java
minimumPxVersionCCMJAVA, _ = version.NewVersion("2.8")
Expand Down Expand Up @@ -1031,6 +1035,11 @@ func ValidateStorageCluster(
return err
}

// Validate Portworx ServiceAccount Token
if err = validatePortworxTokenRefresh(liveCluster, timeout, interval); err != nil {
return err
}

// Validate dmthin
if err = validateDmthinOnPxNodes(liveCluster); err != nil {
return err
Expand Down Expand Up @@ -1490,6 +1499,11 @@ func ValidateUninstallStorageCluster(
return err
}

// Validate deletion of Px ServiceAccount Token Secret
if err := validatePortworxSaTokenSecretDeleted(cluster, timeout, interval); err != nil {
return err
}

// Verify telemetry secret is deleted on UninstallAndWipe when telemetry is enabled
if cluster.Spec.DeleteStrategy != nil && cluster.Spec.DeleteStrategy.Type == corev1.UninstallAndWipeStorageClusterStrategyType {
secret, err := coreops.Instance().GetSecret(TelemetryCertName, cluster.Namespace)
Expand Down Expand Up @@ -1545,6 +1559,33 @@ func validatePortworxConfigMapsDeleted(cluster *corev1.StorageCluster, timeout,
return nil
}

func validatePortworxSaTokenSecretDeleted(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
pxVersion := GetPortworxVersion(cluster)
opVersion, err := GetPxOperatorVersion()
if err != nil {
return err
}
if pxVersion.LessThan(pxVer3_2) || opVersion.LessThan(opVer24_2_0) {
logrus.Infof("pxVersion: %v, opVersion: %v. Skip verification because px token refresh is not supported with these versions.", pxVersion, opVersion)
return nil
}
t := func() (interface{}, bool, error) {
secret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace)
if err != nil {
if errors.IsNotFound(err) {
return nil, false, nil
}
return nil, true, err
}
return nil, true, fmt.Errorf("px ServiceAccount Token Secret exists: %v", secret)
}
if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil {
return err
}
logrus.Debug("Portworx ServiceAccount Token Secret has been deleted successfully")
return nil
}

type podTestFnType func(pod v1.Pod) bool

func validateStorageClusterPods(
Expand Down Expand Up @@ -1794,6 +1835,58 @@ func validatePortworxAPIService(cluster *corev1.StorageCluster, timeout, interva
return nil
}

func validatePortworxTokenRefresh(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
pxVersion := GetPortworxVersion(cluster)
opVersion, err := GetPxOperatorVersion()
if err != nil {
return err
}
if pxVersion.LessThan(pxVer3_2) || opVersion.LessThan(opVer24_2_0) {
logrus.Infof("pxVersion: %v, opVersion: %v. Skip verification because px token refresh is not supported with these versions.", pxVersion, opVersion)
return nil
}
logrus.Infof("Verifying px runc container token...")
// Get one Portworx pod to run commands inside the px runc container on the same node
pxPods, err := coreops.Instance().GetPods(cluster.Namespace, map[string]string{"name": "portworx"})
if err != nil {
return fmt.Errorf("failed to get PX pods, Err: %w", err)
}
pxPod := pxPods.Items[0]
t := func() (interface{}, bool, error) {
pxSaSecret, err := coreops.Instance().GetSecret(pxSaTokenSecretName, cluster.Namespace)
if err != nil {
return nil, true, err
}
expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
if !coreops.Instance().IsPodReady(pxPod) {
return nil, true, fmt.Errorf("[%s] PX pod is not in Ready state to run command inside", pxPod.Name)
}
actualToken, err := runCmdInsidePxPod(&pxPod, "runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token", cluster.Namespace, false)
if err != nil {
return nil, true, err
}
if expectedToken != actualToken {
return nil, true, fmt.Errorf("the token inside px runc container is different from the token in the k8s secret")
}
return actualToken, false, nil
}
token, err := task.DoRetryWithTimeout(t, timeout, interval)
if err != nil {
return err
}
secretList, err := runCmdInsidePxPod(&pxPod, fmt.Sprintf("runc exec portworx "+
"curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+
"--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt", token), cluster.Namespace, false)
if err != nil {
return fmt.Errorf("failed to verify px ServiceAccount token: %w", err)
}
if !strings.Contains(secretList, pxSaTokenSecretName) {
return fmt.Errorf("the secret list returned from k8s api server does not contain %s. Output: %s", pxSaTokenSecretName, secretList)
}
logrus.Infof("token is created and verified: %s", token)
return nil
}

// GetExpectedPxNodeList will get the list of nodes that should be included
// in the given Portworx cluster, by seeing if each non-master node matches the given
// node selectors and affinities.
Expand Down
48 changes: 19 additions & 29 deletions test/integration_test/basic_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -475,45 +475,35 @@ func BasicInstallWithPxSaTokenRefresh(tc *types.TestCase) func(*testing.T) {
cluster, ok := testSpec.(*corev1.StorageCluster)
require.True(t, ok)

verifyTokenFunc := func() string {
verifyTokenRefreshed := func(oldToken string) string {
pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
require.NoError(t, err)
expectedToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
newToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])
require.Eventually(t, func() bool {
actualToken, stderr, err := ci_utils.RunPxCmd("runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/token")
require.Empty(t, stderr)
require.NoError(t, err)
return expectedToken == actualToken
}, 10*time.Minute, 15*time.Second, "the token inside px runc container is different from the token in the k8s secret")

stdout, stderr, err := ci_utils.RunPxCmd(fmt.Sprintf("runc exec portworx "+
"curl -s https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/$(runc exec portworx cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)/secrets "+
"--header 'Authorization: Bearer %s' --cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | grep %s", expectedToken, pxutil.PortworxServiceAccountTokenSecretName))
errMsg := "px not able to communicate with k8s api server with the mounted service account token"
require.True(t, strings.Contains(stdout, pxutil.PortworxServiceAccountTokenSecretName),
fmt.Sprintf("the secret list returned from k8s api server does not contain %s. output: %s", pxutil.PortworxServiceAccountTokenSecretName, stdout))
require.Empty(t, stderr, fmt.Sprintf("%s: %s", errMsg, stderr))
require.NoError(t, err, fmt.Sprintf("%s: %s", errMsg, err.Error()))
logrus.Infof("token is created and verified: %s", expectedToken)
return expectedToken
return oldToken != newToken
}, 10*time.Minute, 15*time.Second, "the token did not get refreshed")
return newToken
}

cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t)
pxSaSecret, err := coreops.Instance().GetSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
require.NoError(t, err)
startupToken := string(pxSaSecret.Data[core.ServiceAccountTokenKey])

logrus.Infof("Verifying px container token...")
token := verifyTokenFunc()
time.Sleep(5 * time.Minute)

time.Sleep(time.Duration(5) * time.Minute)
logrus.Infof("Verifying auto-refreshed px runc container token...")
refreshedToken := verifyTokenFunc()
require.NotEqual(t, token, refreshedToken, "the token did not get refreshed")
refreshedToken := verifyTokenRefreshed(startupToken)
err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "")

logrus.Infof("Verifying px runc container token gets recreated after manual deletion...")
err := coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
err = coreops.Instance().DeleteSecret(pxutil.PortworxServiceAccountTokenSecretName, cluster.Namespace)
require.NoError(t, err)
time.Sleep(time.Duration(2) * time.Minute)
recreatedToken := verifyTokenFunc()
require.NotEqual(t, refreshedToken, recreatedToken, "the token did not get refreshed")
time.Sleep(2 * time.Minute)

verifyTokenRefreshed(refreshedToken)
err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateDeployTimeout, ci_utils.DefaultValidateDeployRetryInterval, true, "")

// Delete and validate the deletion
ci_utils.UninstallAndValidateStorageCluster(cluster, t)
}
}

Expand Down

0 comments on commit 28cc6a7

Please sign in to comment.