From 8b86811074e1aaff68683eb69fd03d42a95f1ac6 Mon Sep 17 00:00:00 2001 From: Mateus Oliveira Date: Sun, 17 Mar 2024 15:37:19 -0300 Subject: [PATCH 1/3] fix: More YAML templates for E2E virt tests Signed-off-by: Mateus Oliveira --- tests/e2e/lib/apps.go | 16 +- tests/e2e/lib/virt_helpers.go | 256 ++++-------------- tests/e2e/lib/virt_storage_helpers.go | 147 +++------- ...cirros-test-disk.yaml => data-volume.yaml} | 9 +- .../openshift-cnv/hyper-converged.yaml | 14 + .../openshift-virtualization.yaml | 27 ++ tests/e2e/virt_backup_restore_suite_test.go | 19 +- 7 files changed, 158 insertions(+), 330 deletions(-) rename tests/e2e/sample-applications/virtual-machines/{cirros-test/cirros-test-disk.yaml => data-volume.yaml} (85%) create mode 100644 tests/e2e/sample-applications/virtual-machines/openshift-cnv/hyper-converged.yaml create mode 100644 tests/e2e/sample-applications/virtual-machines/openshift-cnv/openshift-virtualization.yaml diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 17e111ee59..5ac01936fb 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -10,6 +10,7 @@ import ( "os/exec" "reflect" "sort" + "strings" "time" "github.com/google/go-cmp/cmp" @@ -45,17 +46,26 @@ var ( e2eAppLabelSelector = labels.NewSelector().Add(*e2eAppLabelRequirement) ) -func InstallApplication(ocClient client.Client, file string) error { - return InstallApplicationWithRetries(ocClient, file, 3) +func InstallApplication(ocClient client.Client, file string, replace ...string) error { + return InstallApplicationWithRetries(ocClient, file, 3, replace...) } -func InstallApplicationWithRetries(ocClient client.Client, file string, retries int) error { +func InstallApplicationWithRetries(ocClient client.Client, file string, retries int, replace ...string) error { template, err := os.ReadFile(file) if err != nil { return err } obj := &unstructured.UnstructuredList{} + for index, replacement := range replace { + replaceInTemplate := fmt.Sprintf("<>", index+1) + templateRawString := string(template) + if !strings.Contains(templateRawString, replaceInTemplate) { + return fmt.Errorf("replace directive %s not found in %s template file", replaceInTemplate, file) + } + template = []byte(strings.Replace(templateRawString, replaceInTemplate, replacement, 1)) + } + dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) _, _, err = dec.Decode([]byte(template), nil, obj) if err != nil { diff --git a/tests/e2e/lib/virt_helpers.go b/tests/e2e/lib/virt_helpers.go index f6db15aac2..11f7d95316 100644 --- a/tests/e2e/lib/virt_helpers.go +++ b/tests/e2e/lib/virt_helpers.go @@ -52,53 +52,48 @@ var csvGvr = schema.GroupVersionResource{ Version: "v1alpha1", } +var operatorGroupGvr = schema.GroupVersionResource{ + Group: "operators.coreos.com", + Resource: "operatorgroups", + Version: "v1", +} + type VirtOperator struct { - Client client.Client - Clientset *kubernetes.Clientset - Dynamic dynamic.Interface - Namespace string - Csv string - Version *version.Version + Client client.Client + Clientset *kubernetes.Clientset + Dynamic dynamic.Interface + Namespace string + Csv string + Version *version.Version + OperatorGroup string + Subscription string + HyperConverged string } // GetVirtOperator fills out a new VirtOperator func GetVirtOperator(c client.Client, clientset *kubernetes.Clientset, dynamicClient dynamic.Interface) (*VirtOperator, error) { - namespace := "openshift-cnv" - - csv, operatorVersion, err := getCsvFromPackageManifest(dynamicClient, "kubevirt-hyperconverged") + operatorName := "kubevirt-hyperconverged" + csv, operatorVersion, err := getCsvFromPackageManifest(dynamicClient, operatorName) if err != nil { log.Printf("Failed to get CSV from package manifest") return nil, err } v := &VirtOperator{ - Client: c, - Clientset: clientset, - Dynamic: dynamicClient, - Namespace: namespace, - Csv: csv, - Version: operatorVersion, + Client: c, + Clientset: clientset, + Dynamic: dynamicClient, + Namespace: "openshift-cnv", + Csv: csv, + Version: operatorVersion, + OperatorGroup: "kubevirt-hyperconverged-group", + Subscription: "hco-operatorhub", + HyperConverged: operatorName, } return v, nil } -// Helper to create an operator group object, common to installOperatorGroup -// and removeOperatorGroup. -func (v *VirtOperator) makeOperatorGroup() *operatorsv1.OperatorGroup { - return &operatorsv1.OperatorGroup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubevirt-hyperconverged-group", - Namespace: v.Namespace, - }, - Spec: operatorsv1.OperatorGroupSpec{ - TargetNamespaces: []string{ - v.Namespace, - }, - }, - } -} - // getCsvFromPackageManifest returns the current CSV from the first channel // in the given PackageManifest name. Uses the dynamic client because adding // the real PackageManifest API from OLM was actually more work than this. @@ -168,14 +163,14 @@ func (v *VirtOperator) checkNamespace() bool { // Checks for the existence of the virtualization operator group func (v *VirtOperator) checkOperatorGroup() bool { group := operatorsv1.OperatorGroup{} - err := v.Client.Get(context.TODO(), client.ObjectKey{Namespace: v.Namespace, Name: "kubevirt-hyperconverged-group"}, &group) + err := v.Client.Get(context.TODO(), client.ObjectKey{Namespace: v.Namespace, Name: v.OperatorGroup}, &group) return err == nil } // Checks if there is a virtualization subscription func (v *VirtOperator) checkSubscription() bool { subscription := operatorsv1alpha1.Subscription{} - err := v.Client.Get(context.TODO(), client.ObjectKey{Namespace: v.Namespace, Name: "hco-operatorhub"}, &subscription) + err := v.Client.Get(context.TODO(), client.ObjectKey{Namespace: v.Namespace, Name: v.Subscription}, &subscription) return err == nil } @@ -195,7 +190,7 @@ func (v *VirtOperator) checkCsv() bool { // health status field is "healthy". Uses dynamic client to avoid uprooting lots // of package dependencies, which should probably be fixed later. func (v *VirtOperator) checkHco() bool { - unstructuredHco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Get(context.Background(), "kubevirt-hyperconverged", metav1.GetOptions{}) + unstructuredHco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Get(context.Background(), v.HyperConverged, metav1.GetOptions{}) if err != nil { log.Printf("Error getting HCO: %v", err) return false @@ -217,7 +212,8 @@ func (v *VirtOperator) checkHco() bool { // Check if KVM emulation is enabled. func (v *VirtOperator) checkEmulation() bool { - hco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace("openshift-cnv").Get(context.Background(), "kubevirt-hyperconverged", metav1.GetOptions{}) + // TODO check if spec.configuration.developerConfiguration.useEmulation field is set to true in kubevirst object + hco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Get(context.Background(), v.HyperConverged, metav1.GetOptions{}) if err != nil { return false } @@ -241,118 +237,14 @@ func (v *VirtOperator) checkEmulation() bool { return false } -// Creates the target virtualization namespace, likely openshift-cnv or kubevirt-hyperconverged -func (v *VirtOperator) installNamespace() error { - err := v.Client.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: v.Namespace}}) - if err != nil { - log.Printf("Failed to create namespace %s: %v", v.Namespace, err) - return err - } - return nil -} - -// Creates the virtualization operator group -func (v *VirtOperator) installOperatorGroup() error { - group := v.makeOperatorGroup() - err := v.Client.Create(context.Background(), group) - if err != nil { - if !strings.Contains(err.Error(), "already exists") { - log.Printf("Failed to create operator group: %v", err) - return err - } - } - return nil -} - -// Creates the subscription, which triggers creation of the ClusterServiceVersion. -func (v *VirtOperator) installSubscription() error { - subscription := &operatorsv1alpha1.Subscription{ - ObjectMeta: metav1.ObjectMeta{ - Name: "hco-operatorhub", - Namespace: v.Namespace, - }, - Spec: &operatorsv1alpha1.SubscriptionSpec{ - CatalogSource: "redhat-operators", - CatalogSourceNamespace: "openshift-marketplace", - Package: "kubevirt-hyperconverged", - Channel: "stable", - StartingCSV: v.Csv, - InstallPlanApproval: operatorsv1alpha1.ApprovalAutomatic, - }, - } - err := v.Client.Create(context.Background(), subscription) - if err != nil { - log.Printf("Failed to create subscription: %v", err) - return err - } - - return nil -} - -// Creates a HyperConverged Operator instance. Another dynamic client to avoid -// bringing in the KubeVirt APIs for now. -func (v *VirtOperator) installHco() error { - unstructuredHco := unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "hco.kubevirt.io/v1beta1", - "kind": "HyperConverged", - "metadata": map[string]interface{}{ - "name": "kubevirt-hyperconverged", - "namespace": v.Namespace, - }, - "spec": map[string]interface{}{}, - }, - } - _, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Create(context.Background(), &unstructuredHco, metav1.CreateOptions{}) - if err != nil { - log.Printf("Error creating HCO: %v", err) - return err - } - - return nil -} - -func (v *VirtOperator) configureEmulation() error { - hco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace("openshift-cnv").Get(context.Background(), "kubevirt-hyperconverged", metav1.GetOptions{}) - if err != nil { - return err - } - if hco == nil { - return fmt.Errorf("could not find hyperconverged operator to set emulation annotation") - } - - annotations, ok, err := unstructured.NestedMap(hco.UnstructuredContent(), "metadata", "annotations") - if err != nil { - return err - } - if !ok { - annotations = make(map[string]interface{}) - } - annotations[emulationAnnotation] = useEmulation - - if err := unstructured.SetNestedMap(hco.UnstructuredContent(), annotations, "metadata", "annotations"); err != nil { - return err - } - - _, err = v.Dynamic.Resource(hyperConvergedGvr).Namespace("openshift-cnv").Update(context.Background(), hco, metav1.UpdateOptions{}) - if err != nil { - return err - } - - return nil -} - // Creates target namespace if needed, and waits for it to exist func (v *VirtOperator) ensureNamespace(timeout time.Duration) error { if !v.checkNamespace() { - if err := v.installNamespace(); err != nil { - return err - } err := wait.PollImmediate(time.Second, timeout, func() (bool, error) { return v.checkNamespace(), nil }) if err != nil { - return fmt.Errorf("timed out waiting to create namespace %s: %w", v.Namespace, err) + return fmt.Errorf("timed out waiting to for namespace %s: %w", v.Namespace, err) } } else { log.Printf("Namespace %s already present, no action required", v.Namespace) @@ -364,17 +256,14 @@ func (v *VirtOperator) ensureNamespace(timeout time.Duration) error { // Creates operator group if needed, and waits for it to exist func (v *VirtOperator) ensureOperatorGroup(timeout time.Duration) error { if !v.checkOperatorGroup() { - if err := v.installOperatorGroup(); err != nil { - return err - } err := wait.PollImmediate(time.Second, timeout, func() (bool, error) { return v.checkOperatorGroup(), nil }) if err != nil { - return fmt.Errorf("timed out waiting to create operator group kubevirt-hyperconverged-group: %w", err) + return fmt.Errorf("timed out waiting to create operator group %s: %w", v.OperatorGroup, err) } } else { - log.Printf("Operator group already present, no action required") + log.Printf("Operator group %s already present, no action required", v.OperatorGroup) } return nil @@ -383,17 +272,14 @@ func (v *VirtOperator) ensureOperatorGroup(timeout time.Duration) error { // Creates the virtualization subscription if needed, and waits for it to exist func (v *VirtOperator) ensureSubscription(timeout time.Duration) error { if !v.checkSubscription() { - if err := v.installSubscription(); err != nil { - return err - } err := wait.PollImmediate(time.Second, timeout, func() (bool, error) { return v.checkSubscription(), nil }) if err != nil { - return fmt.Errorf("timed out waiting to create subscription: %w", err) + return fmt.Errorf("timed out waiting to create subscription %s: %w", v.Subscription, err) } } else { - log.Printf("Subscription already created, no action required") + log.Printf("Subscription %s already present, no action required", v.Subscription) } return nil @@ -413,9 +299,6 @@ func (v *VirtOperator) ensureCsv(timeout time.Duration) error { // Creates HyperConverged Operator instance if needed, and waits for it to go healthy func (v *VirtOperator) ensureHco(timeout time.Duration) error { if !v.checkHco() { - if err := v.installHco(); err != nil { - return err - } err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { return v.checkHco(), nil }) @@ -423,7 +306,7 @@ func (v *VirtOperator) ensureHco(timeout time.Duration) error { return fmt.Errorf("timed out waiting to create HCO: %w", err) } } else { - log.Printf("HCO already created, no action required") + log.Printf("HCO already present, no action required") } return nil @@ -441,12 +324,7 @@ func (v *VirtOperator) removeNamespace() error { // Deletes the virtualization operator group func (v *VirtOperator) removeOperatorGroup() error { - group := v.makeOperatorGroup() - err := v.Client.Delete(context.Background(), group) - if err != nil { - return err - } - return nil + return v.Dynamic.Resource(operatorGroupGvr).Namespace(v.Namespace).Delete(context.Background(), v.OperatorGroup, metav1.DeleteOptions{}) } // Deletes the kubvirt subscription @@ -465,7 +343,7 @@ func (v *VirtOperator) removeCsv() error { // Deletes a HyperConverged Operator instance. func (v *VirtOperator) removeHco() error { - err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Delete(context.Background(), "kubevirt-hyperconverged", metav1.DeleteOptions{}) + err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Delete(context.Background(), v.HyperConverged, metav1.DeleteOptions{}) if err != nil { log.Printf("Error deleting HCO: %v", err) return err @@ -644,38 +522,12 @@ func (v *VirtOperator) ensureVmRemoval(namespace, name string, timeout time.Dura // Enable KVM emulation for use on cloud clusters that do not have direct // access to the host server's virtualization capabilities. -func (v *VirtOperator) EnsureEmulation(timeout time.Duration) error { +func (v *VirtOperator) ensureEmulation() error { if v.checkEmulation() { - log.Printf("KVM emulation already enabled, no work needed to turn it on.") return nil } - log.Printf("Enabling KVM emulation...") - - // Retry if there are API server conflicts ("the object has been modified") - timeTaken := 0 * time.Second - err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { - timeTaken += 5 - innerErr := v.configureEmulation() - if innerErr != nil { - if apierrors.IsConflict(innerErr) { - log.Printf("HCO modification conflict, trying again...") - return false, nil // Conflict: try again - } - return false, innerErr // Anything else: give up - } - return innerErr == nil, nil - }) - if err != nil { - return err - } - - timeout = timeout - timeTaken - err = wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { - return v.checkEmulation(), nil - }) - - return err + return fmt.Errorf("KVM emulation is not enabled in hyperconverged %s", v.HyperConverged) } // IsVirtInstalled returns whether or not the OpenShift Virtualization operator @@ -696,23 +548,23 @@ func (v *VirtOperator) EnsureVirtInstallation() error { return nil } - log.Printf("Creating virtualization namespace %s", v.Namespace) + log.Printf("Creating virtualization namespace %s, operator group %s and virtualization operator subscription %s", v.Namespace, v.OperatorGroup, v.Subscription) + installOperatorTemplate := filepath.Join("sample-applications", "virtual-machines", v.Namespace, "openshift-virtualization.yaml") + if err := InstallApplication(v.Client, installOperatorTemplate, v.Csv); err != nil { + return err + } + if err := v.ensureNamespace(10 * time.Second); err != nil { return err } - log.Printf("Created namespace %s", v.Namespace) - log.Printf("Creating operator group kubevirt-hyperconverged-group") if err := v.ensureOperatorGroup(10 * time.Second); err != nil { return err } - log.Println("Created operator group") - log.Printf("Creating virtualization operator subscription") if err := v.ensureSubscription(10 * time.Second); err != nil { return err } - log.Println("Created subscription") log.Printf("Waiting for ClusterServiceVersion") if err := v.ensureCsv(5 * time.Minute); err != nil { @@ -720,12 +572,22 @@ func (v *VirtOperator) EnsureVirtInstallation() error { } log.Println("CSV ready") - log.Printf("Creating hyperconverged operator") + log.Printf("Creating hyperconverged %s", v.HyperConverged) + hyperConvergedTemplate := filepath.Join("sample-applications", "virtual-machines", v.Namespace, "hyper-converged.yaml") + if err := InstallApplication(v.Client, hyperConvergedTemplate); err != nil { + return err + } if err := v.ensureHco(5 * time.Minute); err != nil { return err } log.Printf("Created HCO") + log.Printf("Checking if KVM emulation is enabled in hyperconverged %s", v.HyperConverged) + if err := v.ensureEmulation(); err != nil { + return err + } + log.Printf("KVM emulation is enabled in hyperconverged %s", v.HyperConverged) + return nil } @@ -749,7 +611,7 @@ func (v *VirtOperator) EnsureVirtRemoval() error { } log.Println("CSV removed") - log.Printf("Deleting operator group kubevirt-hyperconverged-group") + log.Printf("Deleting operator group %s", v.OperatorGroup) if err := v.ensureOperatorGroupRemoved(10 * time.Second); err != nil { return err } diff --git a/tests/e2e/lib/virt_storage_helpers.go b/tests/e2e/lib/virt_storage_helpers.go index 7e3e6b174c..aca4753fc5 100644 --- a/tests/e2e/lib/virt_storage_helpers.go +++ b/tests/e2e/lib/virt_storage_helpers.go @@ -2,10 +2,10 @@ package lib import ( "context" + "encoding/json" "fmt" "log" "path/filepath" - "strings" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -114,114 +114,44 @@ func (v *VirtOperator) checkDataVolumeReady(namespace, name string) bool { return phase == "Succeeded" } -func (v *VirtOperator) getDataVolumeSize(namespace, name string) (string, error) { - unstructuredDataVolume, err := v.getDataVolume(namespace, name) - if err != nil { - log.Printf("Error getting DataVolume %s/%s: %v", namespace, name, err) - return "", err - } - if unstructuredDataVolume == nil { - return "", err - } - size, ok, err := unstructured.NestedString(unstructuredDataVolume.UnstructuredContent(), "spec", "pvc", "resources", "requests", "storage") - if err != nil { - log.Printf("Error getting size from DataVolume: %v", err) - return "", err - } - if !ok { - return "", err - } - return size, nil -} - -// Create a DataVolume, accepting an unstructured source specification. -// Also add annotations to immediately create and bind to a PersistentVolume, -// and to avoid deleting the DataVolume after the PVC is all ready. -func (v *VirtOperator) createDataVolumeFromSource(namespace, name, size string, source map[string]interface{}) error { - unstructuredDataVolume := unstructured.Unstructured{ - Object: map[string]interface{}{ - "apiVersion": "cdi.kubevirt.io/v1beta1", - "kind": "DataVolume", - "metadata": map[string]interface{}{ - "name": name, - "namespace": namespace, - "annotations": map[string]interface{}{ - "cdi.kubevirt.io/storage.bind.immediate.requested": "", - "cdi.kubevirt.io/storage.deleteAfterCompletion": "false", - }, - }, - "spec": map[string]interface{}{ - "source": source, - "pvc": map[string]interface{}{ - "accessModes": []string{ - "ReadWriteOnce", - }, - "resources": map[string]interface{}{ - "requests": map[string]interface{}{ - "storage": size, - }, - }, - }, - }, - }, - } - - _, err := v.Dynamic.Resource(dataVolumeGVK).Namespace(namespace).Create(context.Background(), &unstructuredDataVolume, metav1.CreateOptions{}) - if err != nil { - if apierrors.IsAlreadyExists(err) { - return nil - } - if strings.Contains(err.Error(), "already exists") { - return nil - } - log.Printf("Error creating DataVolume: %v", err) - return err - } - - return nil -} - // Create a DataVolume and ask it to fill itself with the contents of the given URL. -func (v *VirtOperator) createDataVolumeFromUrl(namespace, name, url, size string) error { +func (v *VirtOperator) createDataVolumeFromUrl(name, url string, timeout time.Duration) error { urlSource := map[string]interface{}{ "http": map[string]interface{}{ "url": url, }, } - return v.createDataVolumeFromSource(namespace, name, size, urlSource) + urlSourceJson, err := json.Marshal(urlSource) + if err != nil { + return err + } + return v.CreateDataVolume(v.Namespace, name, string(urlSourceJson), timeout) } // Create a DataVolume as a clone of an existing PVC. -func (v *VirtOperator) createDataVolumeFromPvc(sourceNamespace, sourceName, cloneNamespace, cloneName, size string) error { +func (v *VirtOperator) createDataVolumeFromPvc(sourceName, cloneNamespace, cloneName string, timeout time.Duration) error { pvcSource := map[string]interface{}{ "pvc": map[string]interface{}{ "name": sourceName, - "namespace": sourceNamespace, + "namespace": v.Namespace, }, } - return v.createDataVolumeFromSource(cloneNamespace, cloneName, size, pvcSource) + pvcSourceJson, err := json.Marshal(pvcSource) + if err != nil { + return err + } + return v.CreateDataVolume(cloneNamespace, cloneName, string(pvcSourceJson), timeout) } // Create a DataVolume and wait for it to be ready. -func (v *VirtOperator) EnsureDataVolumeFromUrl(namespace, name, url, size string, timeout time.Duration) error { - if !v.checkDataVolumeExists(namespace, name) { - if err := v.createDataVolumeFromUrl(namespace, name, url, size); err != nil { +func (v *VirtOperator) EnsureDataVolumeFromUrl(name, url string, timeout time.Duration) error { + if !v.checkDataVolumeExists(v.Namespace, name) { + if err := v.createDataVolumeFromUrl(name, url, timeout); err != nil { return err } - log.Printf("Created DataVolume %s/%s from %s", namespace, name, url) - } else { - log.Printf("DataVolume %s/%s already created, checking for readiness", namespace, name) - } - - err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { - return v.checkDataVolumeReady(namespace, name), nil - }) - if err != nil { - return fmt.Errorf("timed out waiting for DataVolume %s/%s to go ready: %w", namespace, name, err) + log.Printf("Created DataVolume %s/%s from %s", v.Namespace, name, url) } - log.Printf("DataVolume %s/%s ready", namespace, name) - return nil } @@ -300,45 +230,31 @@ func (v *VirtOperator) RemovePvc(namespace, name string, timeout time.Duration) } // Clone a DataVolume and wait for the copy to be ready. -func (v *VirtOperator) CloneDisk(sourceNamespace, sourceName, cloneNamespace, cloneName string, timeout time.Duration) error { - log.Printf("Cloning %s/%s to %s/%s...", sourceNamespace, sourceName, cloneNamespace, cloneName) - if !v.checkDataVolumeExists(sourceNamespace, sourceName) { +func (v *VirtOperator) CloneDisk(sourceName, cloneNamespace, cloneName string, timeout time.Duration) error { + log.Printf("Cloning %s/%s to %s/%s...", v.Namespace, sourceName, cloneNamespace, cloneName) + if !v.checkDataVolumeExists(v.Namespace, sourceName) { return fmt.Errorf("source disk does not exist") } - size, err := v.getDataVolumeSize(sourceNamespace, sourceName) - if err != nil { - return fmt.Errorf("failed to get disk size for clone: %w", err) - } - - if err := v.createDataVolumeFromPvc(sourceNamespace, sourceName, cloneNamespace, cloneName, size); err != nil { + if err := v.createDataVolumeFromPvc(sourceName, cloneNamespace, cloneName, timeout); err != nil { return fmt.Errorf("failed to clone disk: %w", err) } - err = wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { - return v.checkDataVolumeReady(cloneNamespace, cloneName), nil - }) - if err != nil { - return fmt.Errorf("timed out waiting to clone DataVolume %s/%s to %s/%s: %w", sourceNamespace, sourceName, cloneNamespace, cloneName, err) - } - return nil } -// Create a DataVolume from a sample YAML template. The namespace argument -// should match a subdirectory under sample-applications/virtual-machines, and -// the name argument should match a .yaml file in that directory, for example: -// -// sample-applications/virtual-machines/example-vm-test/example-vm-test-disk.yaml +// Create a DataVolume from sample-applications/virtual-machines/data-volume.yaml +// template, with specified name, namespace and source. // -// This file must specify a DataVolume with the following annotations set: +// The template must specify a DataVolume with the following annotations set: // // cdi.kubevirt.io/storage.bind.immediate.requested: "" // cdi.kubevirt.io/storage.deleteAfterCompletion: "false" // // This function will then wait for that DataVolume to be marked "Succeeded". -func (v *VirtOperator) CreateDiskFromYaml(namespace, name string, timeout time.Duration) error { - if err := InstallApplication(v.Client, filepath.Join("sample-applications", "virtual-machines", namespace, name+".yaml")); err != nil { +func (v *VirtOperator) CreateDataVolume(namespace, name, source string, timeout time.Duration) error { + dataVolumeTemplate := filepath.Join("sample-applications", "virtual-machines", "data-volume.yaml") + if err := InstallApplication(v.Client, dataVolumeTemplate, name, namespace, source); err != nil { return fmt.Errorf("failed to create DataVolume %s/%s: %w", namespace, name, err) } @@ -346,5 +262,10 @@ func (v *VirtOperator) CreateDiskFromYaml(namespace, name string, timeout time.D return v.checkDataVolumeReady(namespace, name), nil }) - return err + if err != nil { + return fmt.Errorf("timed out waiting for DataVolume %s/%s to go ready: %w", namespace, name, err) + } + + log.Printf("DataVolume %s/%s ready", namespace, name) + return nil } diff --git a/tests/e2e/sample-applications/virtual-machines/cirros-test/cirros-test-disk.yaml b/tests/e2e/sample-applications/virtual-machines/data-volume.yaml similarity index 85% rename from tests/e2e/sample-applications/virtual-machines/cirros-test/cirros-test-disk.yaml rename to tests/e2e/sample-applications/virtual-machines/data-volume.yaml index 7719ecf233..d6c68537a3 100644 --- a/tests/e2e/sample-applications/virtual-machines/cirros-test/cirros-test-disk.yaml +++ b/tests/e2e/sample-applications/virtual-machines/data-volume.yaml @@ -4,8 +4,8 @@ items: - apiVersion: cdi.kubevirt.io/v1beta1 kind: DataVolume metadata: - name: cirros-test-disk - namespace: cirros-test + name: <> + namespace: <> annotations: # The test code wants to watch a DataVolume for the status of a download # or clone. CDI defaults to deleting a DataVolume some time after it is @@ -16,10 +16,7 @@ items: cdi.kubevirt.io/storage.bind.immediate.requested: "" cdi.kubevirt.io/storage.deleteAfterCompletion: "false" spec: - source: - pvc: - name: cirros-dv - namespace: openshift-cnv + source: <> pvc: accessModes: - ReadWriteOnce diff --git a/tests/e2e/sample-applications/virtual-machines/openshift-cnv/hyper-converged.yaml b/tests/e2e/sample-applications/virtual-machines/openshift-cnv/hyper-converged.yaml new file mode 100644 index 0000000000..0625f0bb69 --- /dev/null +++ b/tests/e2e/sample-applications/virtual-machines/openshift-cnv/hyper-converged.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: List +items: + - apiVersion: hco.kubevirt.io/v1beta1 + kind: HyperConverged + metadata: + name: kubevirt-hyperconverged + namespace: openshift-cnv + annotations: + # TODO this triggers Alert KubevirtHyperconvergedClusterOperatorUSModification: unsafe modification for + # the kubevirt.kubevirt.io/jsonpatch annotation in the HyperConverged resource. + kubevirt.kubevirt.io/jsonpatch: > + [{"op": "add", "path": "/spec/configuration/developerConfiguration", "value": {"useEmulation": true}}] + spec: {} diff --git a/tests/e2e/sample-applications/virtual-machines/openshift-cnv/openshift-virtualization.yaml b/tests/e2e/sample-applications/virtual-machines/openshift-cnv/openshift-virtualization.yaml new file mode 100644 index 0000000000..77d77d5365 --- /dev/null +++ b/tests/e2e/sample-applications/virtual-machines/openshift-cnv/openshift-virtualization.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: List +items: + - apiVersion: v1 + kind: Namespace + metadata: + name: openshift-cnv + - apiVersion: operators.coreos.com/v1 + kind: OperatorGroup + metadata: + name: kubevirt-hyperconverged-group + namespace: openshift-cnv + spec: + targetNamespaces: + - openshift-cnv + - apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: hco-operatorhub + namespace: openshift-cnv + spec: + source: redhat-operators + sourceNamespace: openshift-marketplace + name: kubevirt-hyperconverged + channel: "stable" + startingCSV: <> + installPlanApproval: Automatic diff --git a/tests/e2e/virt_backup_restore_suite_test.go b/tests/e2e/virt_backup_restore_suite_test.go index 0694cd4106..e2e733636a 100644 --- a/tests/e2e/virt_backup_restore_suite_test.go +++ b/tests/e2e/virt_backup_restore_suite_test.go @@ -35,10 +35,11 @@ func getLatestCirrosImageURL() (string, error) { return imageURL, nil } +const cirrOSDataVolume = "cirros-dv" + type VmBackupRestoreCase struct { BackupRestoreCase - Source string - SourceNamespace string + Source string } func runVmBackupAndRestore(brCase VmBackupRestoreCase, expectedErr error, updateLastBRcase func(brCase VmBackupRestoreCase), updateLastInstallTime func(), v *lib.VirtOperator) { @@ -62,7 +63,7 @@ func runVmBackupAndRestore(brCase VmBackupRestoreCase, expectedErr error, update // something before running whatever import process it has been asked to // run, so this disk also needs the storage.bind.immediate.requested // annotation to get it to start the cloning process. - err = v.CreateDiskFromYaml(brCase.Namespace, diskName, 5*time.Minute) + err = v.CloneDisk(brCase.Source, brCase.Namespace, diskName, 5*time.Minute) gomega.Expect(err).To(gomega.BeNil()) err = v.CreateVm(brCase.Namespace, vmName, 5*time.Minute) @@ -120,19 +121,16 @@ var _ = ginkgov2.Describe("VM backup and restore tests", ginkgov2.Ordered, func( wasInstalledFromTest = true } - err = v.EnsureEmulation(20 * time.Second) - gomega.Expect(err).To(gomega.BeNil()) - url, err := getLatestCirrosImageURL() gomega.Expect(err).To(gomega.BeNil()) - err = v.EnsureDataVolumeFromUrl("openshift-cnv", "cirros-dv", url, "128Mi", 5*time.Minute) + err = v.EnsureDataVolumeFromUrl(cirrOSDataVolume, url, 5*time.Minute) gomega.Expect(err).To(gomega.BeNil()) dpaCR.CustomResource.Spec.Configuration.Velero.DefaultPlugins = append(dpaCR.CustomResource.Spec.Configuration.Velero.DefaultPlugins, v1alpha1.DefaultPluginKubeVirt) }) var _ = ginkgov2.AfterAll(func() { - v.RemoveDataVolume("openshift-cnv", "cirros-dv", 2*time.Minute) + v.RemoveDataVolume(v.Namespace, cirrOSDataVolume, 2*time.Minute) if v != nil && wasInstalledFromTest { v.EnsureVirtRemoval() @@ -148,9 +146,8 @@ var _ = ginkgov2.Describe("VM backup and restore tests", ginkgov2.Ordered, func( runVmBackupAndRestore(brCase, expectedError, updateLastBRcase, updateLastInstallTime, v) }, - ginkgov2.Entry("default virtual machine backup and restore", ginkgov2.Label("virt"), VmBackupRestoreCase{ - Source: "cirros-dv", - SourceNamespace: "openshift-cnv", + ginkgov2.Entry("CirrOS virtual machine backup and restore", ginkgov2.Label("virt"), VmBackupRestoreCase{ + Source: cirrOSDataVolume, BackupRestoreCase: BackupRestoreCase{ Namespace: "cirros-test", Name: "cirros-test", From 01d60eb3597098524f9290612ba2d6d5459d71ad Mon Sep 17 00:00:00 2001 From: Mateus Oliveira Date: Sun, 17 Mar 2024 17:18:04 -0300 Subject: [PATCH 2/3] fixup! fix: More YAML templates for E2E virt tests Signed-off-by: Mateus Oliveira --- tests/e2e/lib/virt_helpers.go | 8 ++++---- .../virtual-machines/openshift-cnv/hyper-converged.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/lib/virt_helpers.go b/tests/e2e/lib/virt_helpers.go index 11f7d95316..e173393028 100644 --- a/tests/e2e/lib/virt_helpers.go +++ b/tests/e2e/lib/virt_helpers.go @@ -215,9 +215,11 @@ func (v *VirtOperator) checkEmulation() bool { // TODO check if spec.configuration.developerConfiguration.useEmulation field is set to true in kubevirst object hco, err := v.Dynamic.Resource(hyperConvergedGvr).Namespace(v.Namespace).Get(context.Background(), v.HyperConverged, metav1.GetOptions{}) if err != nil { + log.Printf("Failed to get hyperConverged: %v", err) return false } if hco == nil { + log.Print("No hyperConverged found") return false } @@ -229,12 +231,10 @@ func (v *VirtOperator) checkEmulation() bool { } if !ok { log.Printf("No KVM emulation annotation (%s) listed on HCO!", emulationAnnotation) - } - if strings.Compare(patcher, useEmulation) == 0 { - return true + return false } - return false + return strings.Compare(patcher, useEmulation) == 0 } // Creates target namespace if needed, and waits for it to exist diff --git a/tests/e2e/sample-applications/virtual-machines/openshift-cnv/hyper-converged.yaml b/tests/e2e/sample-applications/virtual-machines/openshift-cnv/hyper-converged.yaml index 0625f0bb69..e77f69e659 100644 --- a/tests/e2e/sample-applications/virtual-machines/openshift-cnv/hyper-converged.yaml +++ b/tests/e2e/sample-applications/virtual-machines/openshift-cnv/hyper-converged.yaml @@ -9,6 +9,6 @@ items: annotations: # TODO this triggers Alert KubevirtHyperconvergedClusterOperatorUSModification: unsafe modification for # the kubevirt.kubevirt.io/jsonpatch annotation in the HyperConverged resource. - kubevirt.kubevirt.io/jsonpatch: > + kubevirt.kubevirt.io/jsonpatch: >- [{"op": "add", "path": "/spec/configuration/developerConfiguration", "value": {"useEmulation": true}}] spec: {} From df955c5a0142eb66f7706481b51ab159e58715bb Mon Sep 17 00:00:00 2001 From: Mateus Oliveira Date: Tue, 19 Mar 2024 16:52:48 -0300 Subject: [PATCH 3/3] fixup! fix: More YAML templates for E2E virt tests Signed-off-by: Mateus Oliveira --- tests/e2e/lib/apps.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 5ac01936fb..7d5925e949 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "reflect" + "regexp" "sort" "strings" "time" @@ -57,13 +58,27 @@ func InstallApplicationWithRetries(ocClient client.Client, file string, retries } obj := &unstructured.UnstructuredList{} + // YAML templates can have replace directives, in the form of "<>", where N + // is a integer, starting from 1 (increasing by 1, for each new replacement). Replace directives + // can be repeated. The number of replaces must always match the number of unique replace + // directives in template file + uniqueMatches := make(map[string]bool) + for _, match := range regexp.MustCompile(`<>`).FindAll(template, -1) { + uniqueMatches[(string(match))] = true + } + numberOfUniqueMatches := len(uniqueMatches) + numberOfReplaces := len(replace) + if numberOfReplaces != numberOfUniqueMatches { + return fmt.Errorf("number of replaces (%v) differs from number of unique replace directives (%v) in %s template file", numberOfReplaces, numberOfUniqueMatches, file) + } + for index, replacement := range replace { replaceInTemplate := fmt.Sprintf("<>", index+1) templateRawString := string(template) if !strings.Contains(templateRawString, replaceInTemplate) { return fmt.Errorf("replace directive %s not found in %s template file", replaceInTemplate, file) } - template = []byte(strings.Replace(templateRawString, replaceInTemplate, replacement, 1)) + template = []byte(strings.Replace(templateRawString, replaceInTemplate, replacement, -1)) } dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)