added Integration test fpr pdb

pkg/util/test/util.go

@@ -37,6 +37,7 @@ import (
 	k8serrors "github.com/portworx/sched-ops/k8s/errors"
 	openshiftops "github.com/portworx/sched-ops/k8s/openshift"
 	operatorops "github.com/portworx/sched-ops/k8s/operator"
+	policyops "github.com/portworx/sched-ops/k8s/policy"
 	prometheusops "github.com/portworx/sched-ops/k8s/prometheus"
 	rbacops "github.com/portworx/sched-ops/k8s/rbac"
 	"github.com/portworx/sched-ops/task"
@@ -183,6 +184,7 @@ const (
 var TestSpecPath = "testspec"
 
 var (
+	opVer1_5_0, _                     = version.NewVersion("1.5.0-")
 	opVer1_9_1, _                     = version.NewVersion("1.9.1-")
 	opVer1_10, _                      = version.NewVersion("1.10.0-")
 	opVer23_3, _                      = version.NewVersion("23.3.0-")
@@ -195,6 +197,8 @@ var (
 
 	// OCP Dynamic Plugin is only supported in starting with OCP 4.12+ which is k8s v1.25.0+
 	minK8sVersionForDynamicPlugin, _ = version.NewVersion("1.25.0")
+	// PodDisruptionBudget is only supported in starting with k8s v1.21.0+
+	minSupportedK8sVersionForPdb, _ = version.NewVersion("1.21.0")
 
 	pxVer3_0, _  = version.NewVersion("3.0")
 	pxVer2_13, _ = version.NewVersion("2.13")
@@ -1634,6 +1638,11 @@ func validateComponents(pxImageList map[string]string, originalClusterSpec, clus
 		return err
 	}
 
+	// Validate Poddisruptionbudget
+	if err := ValidatePodDisruptionBudegt(cluster, timeout, interval); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -4729,6 +4738,93 @@ func ValidateTelemetryV1Enabled(pxImageList map[string]string, cluster *corev1.S
 	return nil
 }
 
+// ValidatePodDisruptionBudegt validates the value of minavailable and number of disruptions for px-storage poddisruptionbudget
+func ValidatePodDisruptionBudegt(cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
+	logrus.Info("Validate px-storage poddisruptionbudget minAvailable and allowed disruptions")
+
+	kbVer, err := GetK8SVersion()
+	if err != nil {
+		return err
+	}
+	k8sVersion, _ := version.NewVersion(kbVer)
+	opVersion, err := GetPxOperatorVersion()
+	if err != nil {
+		return err
+	}
+
+	// PodDisruptionBudget is supported for k8s version greater than or equal to 1.21 and operator version greater than or equal to 1.5.0
+	if k8sVersion.GreaterThanOrEqual(minSupportedK8sVersionForPdb) && opVersion.GreaterThanOrEqual(opVer1_5_0) {
+		// This is only for non async DR setup
+		t := func() (interface{}, bool, error) {
+
+			nodes, err := operatorops.Instance().ListStorageNodes(cluster.Namespace)
+			if err != nil {
+				return nil, true, fmt.Errorf("failed to get storage nodes: %v", err)
+			}
+
+			nodeslen := 0
+			for _, node := range nodes.Items {
+				if *node.Status.NodeAttributes.Storage {
+					nodeslen++
+				}
+			}
+
+			// Skip PDB validation for px-storage if number of storage nodes is lesser than or equal to 1
+			if nodeslen <= 1 {
+				logrus.Infof("Storage PDB does not exist for storage nodes lesser than or equal to length 1, skipping PDB validattion")
+				return nil, false, nil
+			}
+
+			pdb, err := policyops.Instance().GetPodDisruptionBudget("px-storage", cluster.Namespace)
+			if err != nil {
+				return nil, true, fmt.Errorf("failed to get px-storage poddisruptionbudget, Err: %v", err)
+			}
+			pdbValue := pdb.Spec.MinAvailable.IntValue()
+			allowedDisruptions := pdb.Status.DisruptionsAllowed
+
+			// If annotations to override PDb are not set or if the value of minAvailable is lesser than or equal to 1
+			// or if it is more than or equal to number of storage nodes then use default minAvailable calculation
+			// Else minAvailable will be annotation value
+			checkOverrideValue := false
+			var intPDBVal int
+			if val, ok := cluster.Annotations["portworx.io/pdb-min-available"]; ok {
+				intPDBVal, err = strconv.Atoi(val)
+				if err != nil {
+					return nil, true, fmt.Errorf("error in converting string to int: %v", err)
+				}
+				if intPDBVal < 1 || intPDBVal >= nodeslen {
+					checkOverrideValue = false
+				} else {
+					checkOverrideValue = true
+				}
+
+			}
+
+			if checkOverrideValue {
+				if pdbValue == intPDBVal && allowedDisruptions == int32(nodeslen-intPDBVal) {
+					return nil, false, nil
+				}
+
+			} else {
+				if pdbValue == nodeslen-1 && allowedDisruptions == 1 {
+					return nil, false, nil
+				}
+			}
+
+			return nil, false, fmt.Errorf("incorrect PDB value. Expected %d, Actual %d", nodeslen-1, pdbValue)
+		}
+
+		if _, err := task.DoRetryWithTimeout(t, timeout, interval); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	logrus.Debugf("skipping poddisruptionbudget validation, not supported for kubernetes version %s and operator version %s", k8sVersion, opVersion)
+	return nil
+
+}
+
 // validatePodTopologySpreadConstraints validates pod topology spread constraints
 func validatePodTopologySpreadConstraints(deployment *appsv1.Deployment, timeout, interval time.Duration) error {
 	t := func() (interface{}, bool, error) {

test/integration_test/pdb_test.go

@@ -0,0 +1,119 @@
+package integrationtest
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/go-version"
+	corev1 "github.com/libopenstorage/operator/pkg/apis/core/v1"
+	"github.com/libopenstorage/operator/test/integration_test/types"
+	ci_utils "github.com/libopenstorage/operator/test/integration_test/utils"
+	"github.com/portworx/sched-ops/k8s/operator"
+	"github.com/sirupsen/logrus"
+
+	testutil "github.com/libopenstorage/operator/pkg/util/test"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	minSupportedK8sVersionForPdb, _ = version.NewVersion("1.21.0")
+)
+var testStorageClusterPDBCases = []types.TestCase{
+	{
+		TestName:        "PDBWithStoragelessNode",
+		TestrailCaseIDs: []string{"C58606"},
+		TestSpec: ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{
+			ObjectMeta: metav1.ObjectMeta{Name: "test-stc"},
+		}),
+		ShouldSkip: func(tc *types.TestCase) bool {
+			if len(ci_utils.PxDeviceSpecs) == 0 {
+				logrus.Info("--portworx-device-specs is empty, cannot run PDBWithStoragelessNode test")
+				return true
+			}
+			kbVer, err := testutil.GetK8SVersion()
+			if err != nil {
+				logrus.Info("Skipping PDB test due to :", err)
+				return true
+			}
+			k8sVersion, _ := version.NewVersion(kbVer)
+			return ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer1_5_0) && k8sVersion.LessThan(minSupportedK8sVersionForPdb)
+		},
+		TestFunc: StoragelessNodePDB,
+	},
+	{
+		TestName:        "OverridePDBUsingAnnotation",
+		TestrailCaseIDs: []string{"C58607"},
+		TestSpec: ci_utils.CreateStorageClusterTestSpecFunc(&corev1.StorageCluster{
+			ObjectMeta: metav1.ObjectMeta{Name: "test-stc"},
+		}),
+		ShouldSkip: func(tc *types.TestCase) bool {
+			kbVer, err := testutil.GetK8SVersion()
+			if err != nil {
+				logrus.Info("Skipping PDB test due to :", err)
+				return true
+			}
+			k8sVersion, _ := version.NewVersion(kbVer)
+			return ci_utils.PxOperatorVersion.LessThan(ci_utils.PxOperatorVer1_5_0) && k8sVersion.LessThan(minSupportedK8sVersionForPdb)
+		},
+		TestFunc: OverridePDBUsingAnnotation,
+	},
+}
+
+func StoragelessNodePDB(tc *types.TestCase) func(*testing.T) {
+	return func(t *testing.T) {
+		testSpec := tc.TestSpec(t)
+		cluster, ok := testSpec.(*corev1.StorageCluster)
+		require.True(t, ok)
+
+		// Assuming there are more than 3 nodes in a zone
+		*cluster.Spec.CloudStorage.MaxStorageNodesPerZone = uint32(3)
+		logrus.Info("Validating PDB with storageless nodes using maxstoragenodesperzone value: ", *cluster.Spec.CloudStorage.MaxStorageNodesPerZone)
+		cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t)
+		ci_utils.UninstallAndValidateStorageCluster(cluster, t)
+
+	}
+}
+
+func OverridePDBUsingAnnotation(tc *types.TestCase) func(*testing.T) {
+	return func(t *testing.T) {
+		testSpec := tc.TestSpec(t)
+		cluster, ok := testSpec.(*corev1.StorageCluster)
+		require.True(t, ok)
+		//Add annotations to override the default PDB
+		k8snodecount, err := ci_utils.GetK8sNodeCount()
+		require.NoError(t, err)
+
+		// Override PDB value with (number of k8s nodes -2) to check if allowed disruptions is 2
+		cluster.Annotations = make(map[string]string)
+		logrus.Infof("Validating PDB using minAvailable value: %d", k8snodecount-2)
+		cluster.Annotations["portworx.io/storage-pdb-min-available"] = fmt.Sprintf("%d", k8snodecount-2)
+		cluster = ci_utils.DeployAndValidateStorageCluster(cluster, ci_utils.PxSpecImages, t)
+
+		// Override PDB with value 1 and ensure minAvailable value does not get changed
+		logrus.Infof("Validating PDB using minAvailable value: 1")
+		cluster.Annotations["portworx.io/storage-pdb-min-available"] = "1"
+		cluster, err = ci_utils.UpdateStorageCluster(cluster)
+		require.NoError(t, err)
+		err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateUpgradeTimeout, ci_utils.DefaultValidateUpgradeRetryInterval, true, "")
+		require.NoError(t, err)
+
+		// Override PDB with value equal to storage nodes and ensure minAvailable value does not get changed
+		logrus.Infof("Validating PDB using minAvailable value: %d", k8snodecount)
+		cluster, err = operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace)
+		require.NoError(t, err)
+		cluster.Annotations["portworx.io/storage-pdb-min-available"] = fmt.Sprintf("%d", k8snodecount)
+		cluster, err = ci_utils.UpdateStorageCluster(cluster)
+		require.NoError(t, err)
+		err = testutil.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateUpgradeTimeout, ci_utils.DefaultValidateUpgradeRetryInterval, true, "")
+		require.NoError(t, err)
+
+		ci_utils.UninstallAndValidateStorageCluster(cluster, t)
+	}
+}
+
+func TestStorageClusterPDB(t *testing.T) {
+	for _, tc := range testStorageClusterPDBCases {
+		tc.RunTest(t)
+	}
+}

test/integration_test/utils/px_operator.go

@@ -20,6 +20,8 @@ const (
 )
 
 var (
+	// PxOperator1_5_0 portworx-operator 1.5.0 minimum version
+	PxOperatorVer1_5_0, _ = version.NewVersion("1.5.0-")
 	// PxOperatorVer1_7 portworx-operator 1.7 minimum version
 	PxOperatorVer1_7, _ = version.NewVersion("1.7-")
 	// PxOperatorVer1_8 portworx-operator 1.8 minimum version

test/integration_test/utils/storagecluster.go

@@ -121,6 +121,17 @@ func ConstructStorageCluster(cluster *corev1.StorageCluster, specGenURL string,
 			}
 		}
 	}
+	// If not Ocp but cloud provider is Vsphere, add secret and other env vars
+	if !IsOcp && CloudProvider == cloudops.Vsphere {
+		EnvVarCreds, err := testutil.CreateVsphereCredentialEnvVarsFromSecret(cluster.Namespace)
+		if err != nil {
+			return err
+		}
+
+		if EnvVarCreds != nil {
+			envVarList = append(envVarList, EnvVarCreds...)
+		}
+	}
 
 	// Add EKS annotation
 	if IsEks {
@@ -497,3 +508,12 @@ func ValidateStorageClusterComponents(cluster *corev1.StorageCluster) error {
 	// TODO: Validate the components are running with expected configuration
 	return nil
 }
+
+func GetK8sNodeCount() (int, error) {
+	nodes, err := schedopsCore.Instance().GetNodes()
+	if err != nil {
+		return -1, err
+	}
+	return len(nodes.Items), nil
+
+}