From 57345ba7a11df6941997fbd5253af71424e8424c Mon Sep 17 00:00:00 2001 From: svijaykumar-px Date: Wed, 21 Feb 2024 09:49:49 +0000 Subject: [PATCH] added Integration test fpr pdb --- pkg/util/test/util.go | 96 ++++++++++++++ test/integration_test/pdb_test.go | 119 ++++++++++++++++++ test/integration_test/utils/px_operator.go | 2 + test/integration_test/utils/storagecluster.go | 20 +++ 4 files changed, 237 insertions(+) create mode 100644 test/integration_test/pdb_test.go diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 6f5f6b36b..8fb4b0a64 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -37,6 +37,7 @@ import ( k8serrors "github.com/portworx/sched-ops/k8s/errors" openshiftops "github.com/portworx/sched-ops/k8s/openshift" operatorops "github.com/portworx/sched-ops/k8s/operator" + policyops "github.com/portworx/sched-ops/k8s/policy" prometheusops "github.com/portworx/sched-ops/k8s/prometheus" rbacops "github.com/portworx/sched-ops/k8s/rbac" "github.com/portworx/sched-ops/task" @@ -183,6 +184,7 @@ const ( var TestSpecPath = "testspec" var ( + opVer1_5_0, _ = version.NewVersion("1.5.0-") opVer1_9_1, _ = version.NewVersion("1.9.1-") opVer1_10, _ = version.NewVersion("1.10.0-") opVer23_3, _ = version.NewVersion("23.3.0-") @@ -195,6 +197,8 @@ var ( // OCP Dynamic Plugin is only supported in starting with OCP 4.12+ which is k8s v1.25.0+ minK8sVersionForDynamicPlugin, _ = version.NewVersion("1.25.0") + // PodDisruptionBudget is only supported in starting with k8s v1.21.0+ + minSupportedK8sVersionForPdb, _ = version.NewVersion("1.21.0") pxVer3_0, _ = version.NewVersion("3.0") pxVer2_13, _ = version.NewVersion("2.13") @@ -1634,6 +1638,11 @@ func validateComponents(pxImageList map[string]string, originalClusterSpec, clus return err } + // Validate Poddisruptionbudget + if err := ValidatePodDisruptionBudegt(cluster, timeout, interval); err != nil { + return err + } + return nil } @@ -4729,6 +4738,93 @@ func ValidateTelemetryV1Enabled(pxImageList map[string]string, cluster *corev1.S return nil } +// ValidatePodDisruptionBudegt validates the value of minavailable and number of disruptions for px-storage poddisruptionbudget +func ValidatePodDisruptionBudegt(cluster *corev1.StorageCluster, timeout, interval time.Duration) error { + logrus.Info("Validate px-storage poddisruptionbudget minAvailable and allowed disruptions") + + kbVer, err := GetK8SVersion() + if err != nil { + return err + } + k8sVersion, _ := version.NewVersion(kbVer) + opVersion, err := GetPxOperatorVersion() + if err != nil { + return err + } + + // PodDisruptionBudget is supported for k8s version greater than or equal to 1.21 and operator version greater than or equal to 1.5.0 + if k8sVersion.GreaterThanOrEqual(minSupportedK8sVersionForPdb) && opVersion.GreaterThanOrEqual(opVer1_5_0) { + // This is only for non async DR setup + t := func() (interface{}, bool, error) { + + nodes, err := operatorops.Instance().ListStorageNodes(cluster.Namespace) + if err != nil { + return nil, true, fmt.Errorf("failed to get storage nodes: %v", err) + } + + nodeslen := 0 + for _, node := range nodes.Items { + if *node.Status.NodeAttributes.Storage { + nodeslen++ + } + } + + // Skip PDB validation for px-storage if number of storage nodes is lesser than or equal to 1 + if nodeslen <= 1 { + logrus.Infof("Storage PDB does not exist for storage nodes lesser than or equal to length 1, skipping PDB validattion") + return nil, false, nil + } + + pdb, err := policyops.Instance().GetPodDisruptionBudget("px-storage", cluster.Namespace) + if err != nil { + return nil, true, fmt.Errorf("failed to get px-storage poddisruptionbudget, Err: %v", err) + } + pdbValue := pdb.Spec.MinAvailable.IntValue() + allowedDisruptions := pdb.Status.DisruptionsAllowed + + // If annotations to override PDb are not set or if the value of minAvailable is lesser than or equal to 1 + // or if it is more than or equal to number of storage nodes then use default minAvailable calculation + // Else minAvailable will be annotation value + checkOverrideValue := false + var intPDBVal int + if val, ok := cluster.Annotations["portworx.io/pdb-min-available"]; ok { + intPDBVal, err = strconv.Atoi(val) + if err != nil { + return nil, true, fmt.Errorf("error in converting string to int: %v", err) + } + if intPDBVal < 1 || intPDBVal >= nodeslen { + checkOverrideValue = false + } else { + checkOverrideValue = true + } + + } + + if checkOverrideValue { + if pdbValue == intPDBVal && allowedDisruptions == int32(nodeslen-intPDBVal) { + return nil, false, nil + } + + } else { + if pdbValue == nodeslen-1 && allowedDisruptions == 1 { + return nil, false, nil + } + } + + return nil, false, fmt.Errorf("incorrect PDB value. Expected %d, Actual %d", nodeslen-1, pdbValue) + } + + if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil { + return err + } + return nil + } + + logrus.Debugf("skipping poddisruptionbudget validation, not supported for kubernetes version %s and operator version %s", k8sVersion, opVersion) + return nil + +} + // validatePodTopologySpreadConstraints validates pod topology spread constraints func validatePodTopologySpreadConstraints(deployment *appsv1.Deployment, timeout, interval time.Duration) error { t := func() (interface{}, bool, error) { diff --git a/test/integration_test/pdb_test.go b/test/integration_test/pdb_test.go new file mode 100644 index 000000000..667576721 --- /dev/null +++ b/test/integration_test/pdb_test.go @@ -0,0 +1,119 @@ +package integrationtest + +import ( + "fmt" + "testing" + + "github.com/hashicorp/go-version" + corev1 "github.com/libopenstorage/operator/pkg/apis/core/v1" + "github.com/libopenstorage/operator/test/integration_test/types" + ci_utils "github.com/libopenstorage/operator/test/integration_test/utils" + "github.com/portworx/sched-ops/k8s/operator" + "github.com/sirupsen/logrus" + + testutil "github.com/libopenstorage/operator/pkg/util/test" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + minSupportedK8sVersionForPdb, _ = version.NewVersion("1.21.0") +) +var testStorageClusterPDBCases = []types.TestCase{ + { + TestName: "PDBWithStoragelessNode", + TestrailCaseIDs: []string{"C58606"}, + TestSpec: ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-stc"}, + }), + ShouldSkip: func(tc *types.TestCase) bool { + if len(ci_utils.PxDeviceSpecs) == 0 { + logrus.Info("--portworx-device-specs is empty, cannot run PDBWithStoragelessNode test") + return true + } + kbVer, err := testutil.GetK8SVersion() + if err != nil { + logrus.Info("Skipping PDB test due to :", err) + return true + } + k8sVersion, _ := version.NewVersion(kbVer) + return ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer1_5_0) && k8sVersion.LessThan(minSupportedK8sVersionForPdb) + }, + TestFunc: StoragelessNodePDB, + }, + { + TestName: "OverridePDBUsingAnnotation", + TestrailCaseIDs: []string{"C58607"}, + TestSpec: ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-stc"}, + }), + ShouldSkip: func(tc *types.TestCase) bool { + kbVer, err := testutil.GetK8SVersion() + if err != nil { + logrus.Info("Skipping PDB test due to :", err) + return true + } + k8sVersion, _ := version.NewVersion(kbVer) + return ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer1_5_0) && k8sVersion.LessThan(minSupportedK8sVersionForPdb) + }, + TestFunc: OverridePDBUsingAnnotation, + }, +} + +func StoragelessNodePDB(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + testSpec := tc.TestSpec(t) + cluster, ok := testSpec.(*corev1.StorageCluster) + require.True(t, ok) + + // Assuming there are more than 3 nodes in a zone + *cluster.Spec.CloudStorage.MaxStorageNodesPerZone = uint32(3) + logrus.Info("Validating PDB with storageless nodes using maxstoragenodesperzone value: ", *cluster.Spec.CloudStorage.MaxStorageNodesPerZone) + cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) + ci_utils.UninstallAndValidateStorageCluster(cluster, t) + + } +} + +func OverridePDBUsingAnnotation(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + testSpec := tc.TestSpec(t) + cluster, ok := testSpec.(*corev1.StorageCluster) + require.True(t, ok) + //Add annotations to override the default PDB + k8snodecount, err := ci_utils.GetK8sNodeCount() + require.NoError(t, err) + + // Override PDB value with (number of k8s nodes -2) to check if allowed disruptions is 2 + cluster.Annotations = make(map[string]string) + logrus.Infof("Validating PDB using minAvailable value: %d", k8snodecount-2) + cluster.Annotations["portworx.io/storage-pdb-min-available"] = fmt.Sprintf("%d", k8snodecount-2) + cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t) + + // Override PDB with value 1 and ensure minAvailable value does not get changed + logrus.Infof("Validating PDB using minAvailable value: 1") + cluster.Annotations["portworx.io/storage-pdb-min-available"] = "1" + cluster, err = ci_utils.UpdateStorageCluster(cluster) + require.NoError(t, err) + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateUpgradeTimeout, ci_utils.DefaultValidateUpgradeRetryInterval, true, "") + require.NoError(t, err) + + // Override PDB with value equal to storage nodes and ensure minAvailable value does not get changed + logrus.Infof("Validating PDB using minAvailable value: %d", k8snodecount) + cluster, err = operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace) + require.NoError(t, err) + cluster.Annotations["portworx.io/storage-pdb-min-available"] = fmt.Sprintf("%d", k8snodecount) + cluster, err = ci_utils.UpdateStorageCluster(cluster) + require.NoError(t, err) + err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateUpgradeTimeout, ci_utils.DefaultValidateUpgradeRetryInterval, true, "") + require.NoError(t, err) + + ci_utils.UninstallAndValidateStorageCluster(cluster, t) + } +} + +func TestStorageClusterPDB(t *testing.T) { + for _, tc := range testStorageClusterPDBCases { + tc.RunTest(t) + } +} diff --git a/test/integration_test/utils/px_operator.go b/test/integration_test/utils/px_operator.go index 106a875c6..f2f2ffc71 100644 --- a/test/integration_test/utils/px_operator.go +++ b/test/integration_test/utils/px_operator.go @@ -20,6 +20,8 @@ const ( ) var ( + // PxOperator1_5_0 portworx-operator 1.5.0 minimum version + PxOperatorVer1_5_0, _ = version.NewVersion("1.5.0-") // PxOperatorVer1_7 portworx-operator 1.7 minimum version PxOperatorVer1_7, _ = version.NewVersion("1.7-") // PxOperatorVer1_8 portworx-operator 1.8 minimum version diff --git a/test/integration_test/utils/storagecluster.go b/test/integration_test/utils/storagecluster.go index eaca94c5a..a584c2ee4 100644 --- a/test/integration_test/utils/storagecluster.go +++ b/test/integration_test/utils/storagecluster.go @@ -121,6 +121,17 @@ func ConstructStorageCluster(cluster *corev1.StorageCluster, specGenURL string, } } } + // If not Ocp but cloud provider is Vsphere, add secret and other env vars + if !IsOcp && CloudProvider == cloudops.Vsphere { + EnvVarCreds, err := testutil.CreateVsphereCredentialEnvVarsFromSecret(cluster.Namespace) + if err != nil { + return err + } + + if EnvVarCreds != nil { + envVarList = append(envVarList, EnvVarCreds...) + } + } // Add EKS annotation if IsEks { @@ -497,3 +508,12 @@ func ValidateStorageClusterComponents(cluster *corev1.StorageCluster) error { // TODO: Validate the components are running with expected configuration return nil } + +func GetK8sNodeCount() (int, error) { + nodes, err := schedopsCore.Instance().GetNodes() + if err != nil { + return -1, err + } + return len(nodes.Items), nil + +}