Merge branch '4.20'

Author: DaanHoogland

api/src/main/java/com/cloud/exception/StorageAccessException.java

@@ -26,7 +26,7 @@
 public class StorageAccessException extends RuntimeException {
     private static final long serialVersionUID = SerialVersionUID.StorageAccessException;
 
-    public StorageAccessException(String message) {
-        super(message);
+    public StorageAccessException(String message, Exception causer) {
+        super(message, causer);
     }
 }

api/src/main/java/com/cloud/storage/Storage.java

@@ -16,14 +16,10 @@
 // under the License.
 package com.cloud.storage;
 
+import org.apache.commons.lang.NotImplementedException;
+
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.commons.lang3.StringUtils;
 
 public class Storage {
     public static enum ImageFormat {
@@ -139,6 +135,21 @@ public static enum TemplateType {
         ISODISK /* Template corresponding to a iso (non root disk) present in an OVA */
     }
 
+    public enum EncryptionSupport {
+        /**
+         * Encryption not supported.
+         */
+        Unsupported,
+        /**
+         * Will use hypervisor encryption driver (qemu -> luks)
+         */
+        Hypervisor,
+        /**
+         * Storage pool handles encryption and just provides an encrypted volume
+         */
+        Storage
+    }
+
     /**
      * StoragePoolTypes carry some details about the format and capabilities of a storage pool. While not necessarily a
      * 1:1 with PrimaryDataStoreDriver (and for KVM agent, KVMStoragePool and StorageAdaptor) implementations, it is
@@ -150,61 +161,37 @@ public static enum TemplateType {
      * ensure this is available on the agent side as well. This is best done by defining the StoragePoolType in a common
      * package available on both management server and agent plugin jars.
      */
-    public static class StoragePoolType {
-        private static final Map<String, StoragePoolType> map = new LinkedHashMap<>();
-
-        public static final StoragePoolType Filesystem = new StoragePoolType("Filesystem", false, true, true);
-        public static final StoragePoolType NetworkFilesystem = new StoragePoolType("NetworkFilesystem", true, true, true);
-        public static final StoragePoolType IscsiLUN = new StoragePoolType("IscsiLUN", true, false, false);
-        public static final StoragePoolType Iscsi = new StoragePoolType("Iscsi", true, false, false);
-        public static final StoragePoolType ISO = new StoragePoolType("ISO", false, false, false);
-        public static final StoragePoolType LVM = new StoragePoolType("LVM", false, false, false);
-        public static final StoragePoolType CLVM = new StoragePoolType("CLVM", true, false, false);
-        public static final StoragePoolType RBD = new StoragePoolType("RBD", true, true, false);
-        public static final StoragePoolType SharedMountPoint = new StoragePoolType("SharedMountPoint", true, true, true);
-        public static final StoragePoolType VMFS = new StoragePoolType("VMFS", true, true, false);
-        public static final StoragePoolType PreSetup = new StoragePoolType("PreSetup", true, true, false);
-        public static final StoragePoolType EXT = new StoragePoolType("EXT", false, true, false);
-        public static final StoragePoolType OCFS2 = new StoragePoolType("OCFS2", true, false, false);
-        public static final StoragePoolType SMB = new StoragePoolType("SMB", true, false, false);
-        public static final StoragePoolType Gluster = new StoragePoolType("Gluster", true, false, false);
-        public static final StoragePoolType PowerFlex = new StoragePoolType("PowerFlex", true, true, true);
-        public static final StoragePoolType ManagedNFS = new StoragePoolType("ManagedNFS", true, false, false);
-        public static final StoragePoolType Linstor = new StoragePoolType("Linstor", true, true, false);
-        public static final StoragePoolType DatastoreCluster = new StoragePoolType("DatastoreCluster", true, true, false);
-        public static final StoragePoolType StorPool = new StoragePoolType("StorPool", true,true,true);
-        public static final StoragePoolType FiberChannel = new StoragePoolType("FiberChannel", true,true,false);
-
-
-        private final String name;
+    public static enum StoragePoolType {
+        Filesystem(false, true, EncryptionSupport.Hypervisor), // local directory
+        NetworkFilesystem(true, true, EncryptionSupport.Hypervisor), // NFS
+        IscsiLUN(true, false, EncryptionSupport.Unsupported), // shared LUN, with a clusterfs overlay
+        Iscsi(true, false, EncryptionSupport.Unsupported), // for e.g., ZFS Comstar
+        ISO(false, false, EncryptionSupport.Unsupported), // for iso image
+        LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR
+        CLVM(true, false, EncryptionSupport.Unsupported),
+        RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD
+        SharedMountPoint(true, true, EncryptionSupport.Hypervisor),
+        VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage
+        PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers.
+        EXT(false, true, EncryptionSupport.Unsupported), // XenServer local EXT SR
+        OCFS2(true, false, EncryptionSupport.Unsupported),
+        SMB(true, false, EncryptionSupport.Unsupported),
+        Gluster(true, false, EncryptionSupport.Unsupported),
+        PowerFlex(true, true, EncryptionSupport.Hypervisor), // Dell EMC PowerFlex/ScaleIO (formerly VxFlexOS)
+        ManagedNFS(true, false, EncryptionSupport.Unsupported),
+        Linstor(true, true, EncryptionSupport.Storage),
+        DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters
+        StorPool(true, true, EncryptionSupport.Hypervisor),
+        FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-<wwnvalue>)
+
         private final boolean shared;
         private final boolean overProvisioning;
-        private final boolean encryption;
+        private final EncryptionSupport encryption;
 
-        /**
-         * New StoragePoolType, set the name to check with it in Dao (Note: Do not register it into the map of pool types).
-         * @param name name of the StoragePoolType.
-         */
-        public StoragePoolType(String name) {
-            this.name = name;
-            this.shared = false;
-            this.overProvisioning = false;
-            this.encryption = false;
-        }
-
-        /**
-         * Define a new StoragePoolType, and register it into the map of pool types known to the management server.
-         * @param name Simple unique name of the StoragePoolType.
-         * @param shared Storage pool is shared/accessible to multiple hypervisors
-         * @param overProvisioning Storage pool supports overProvisioning
-         * @param encryption Storage pool supports encrypted volumes
-         */
-        public StoragePoolType(String name, boolean shared, boolean overProvisioning, boolean encryption) {
-            this.name = name;
+        StoragePoolType(boolean shared, boolean overProvisioning, EncryptionSupport encryption) {
             this.shared = shared;
             this.overProvisioning = overProvisioning;
             this.encryption = encryption;
-            addStoragePoolType(this);
         }
 
         public boolean isShared() {
@@ -216,49 +203,11 @@ public boolean supportsOverProvisioning() {
         }
 
         public boolean supportsEncryption() {
-            return encryption;
-        }
-
-        private static void addStoragePoolType(StoragePoolType storagePoolType) {
-            map.putIfAbsent(storagePoolType.name, storagePoolType);
-        }
-
-        public static StoragePoolType[] values() {
-            return map.values().toArray(StoragePoolType[]::new).clone();
-        }
-
-        public static StoragePoolType valueOf(String name) {
-            if (StringUtils.isBlank(name)) {
-                return null;
-            }
-
-            StoragePoolType storage = map.get(name);
-            if (storage == null) {
-                throw new IllegalArgumentException("StoragePoolType '" + name + "' not found");
-            }
-            return storage;
+            return encryption == EncryptionSupport.Hypervisor || encryption == EncryptionSupport.Storage;
         }
 
-        @Override
-        public String toString() {
-            return name;
-        }
-
-        public String name() {
-            return name;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            StoragePoolType that = (StoragePoolType) o;
-            return Objects.equals(name, that.name);
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(name);
+        public EncryptionSupport encryptionSupportMode() {
+            return encryption;
         }
     }
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -63,6 +63,7 @@ public class ApiConstants {
     public static final String BASE64_IMAGE = "base64image";
     public static final String BGP_PEERS = "bgppeers";
     public static final String BGP_PEER_IDS = "bgppeerids";
+    public static final String BATCH_SIZE = "batchsize";
     public static final String BITS = "bits";
     public static final String BOOTABLE = "bootable";
     public static final String BIND_DN = "binddn";
@@ -455,7 +456,6 @@ public class ApiConstants {
     public static final String SENT = "sent";
     public static final String SENT_BYTES = "sentbytes";
     public static final String SERIAL = "serial";
-    public static final String SERVICE_IP = "serviceip";
     public static final String SERVICE_OFFERING_ID = "serviceofferingid";
     public static final String SESSIONKEY = "sessionkey";
     public static final String SHOW_CAPACITIES = "showcapacities";
@@ -485,11 +485,12 @@ public class ApiConstants {
     public static final String STATE = "state";
     public static final String STATS = "stats";
     public static final String STATUS = "status";
-    public static final String STORAGE_TYPE = "storagetype";
-    public static final String STORAGE_POLICY = "storagepolicy";
-    public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
     public static final String STORAGE_CAPABILITIES = "storagecapabilities";
     public static final String STORAGE_CUSTOM_STATS = "storagecustomstats";
+    public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
+    public static final String STORAGE_POLICY = "storagepolicy";
+    public static final String STORAGE_POOL = "storagepool";
+    public static final String STORAGE_TYPE = "storagetype";
     public static final String SUBNET = "subnet";
     public static final String OWNER = "owner";
     public static final String SWAP_OWNER = "swapowner";
@@ -962,7 +963,6 @@ public class ApiConstants {
     public static final String AUTOSCALE_VMGROUP_NAME = "autoscalevmgroupname";
     public static final String BAREMETAL_DISCOVER_NAME = "baremetaldiscovername";
     public static final String BAREMETAL_RCT_URL = "baremetalrcturl";
-    public static final String BATCH_SIZE = "batchsize";
     public static final String UCS_DN = "ucsdn";
     public static final String GSLB_PROVIDER = "gslbprovider";
     public static final String EXCLUSIVE_GSLB_PROVIDER = "isexclusivegslbprovider";
@@ -1210,6 +1210,7 @@ public class ApiConstants {
             "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " +
             "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " +
             "value will be applied.";
+
     public static final String PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS = "The recommended format is \"yyyy-MM-dd'T'HH:mm:ssZ\" (e.g.: \"2023-01-01T12:00:00+0100\"); " +
             "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " +
             "added, it will be interpreted as \"00:00:00\"). If the recommended format is not used, the date will be considered in the server timezone.";
@@ -1218,6 +1219,8 @@ public class ApiConstants {
             "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " +
             "added, it will be interpreted as \"23:59:59\"). If the recommended format is not used, the date will be considered in the server timezone.";
 
+    public static final String VMWARE_DC = "vmwaredc";
+
     /**
      * This enum specifies IO Drivers, each option controls specific policies on I/O.
      * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0).

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -152,7 +152,7 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Deprecated
     @SerializedName("memoryallocated")
     @Param(description = "the amount of the host's memory currently allocated")
-    private long memoryAllocated;
+    private Long memoryAllocated;
 
     @SerializedName("memoryallocatedpercentage")
     @Param(description = "the amount of the host's memory currently allocated in percentage")
@@ -415,7 +415,7 @@ public void setMemWithOverprovisioning(String memWithOverprovisioning){
         this.memWithOverprovisioning=memWithOverprovisioning;
     }
 
-    public void setMemoryAllocated(long memoryAllocated) {
+    public void setMemoryAllocated(Long memoryAllocated) {
         this.memoryAllocated = memoryAllocated;
     }
 
@@ -703,8 +703,8 @@ public Long getMemoryTotal() {
         return memoryTotal;
     }
 
-    public long getMemoryAllocated() {
-        return memoryAllocated;
+    public Long getMemoryAllocated() {
+        return memoryAllocated == null ? 0 : memoryAllocated;
     }
 
     public void setMemoryAllocatedPercentage(String memoryAllocatedPercentage) {

api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java

@@ -74,9 +74,9 @@ public class ManagementServerResponse extends BaseResponse {
     @Param(description = "the running OS kernel version for this Management Server")
     private String kernelVersion;
 
-    @SerializedName(ApiConstants.SERVICE_IP)
+    @SerializedName(ApiConstants.IP_ADDRESS)
     @Param(description = "the IP Address for this Management Server")
-    private String serviceIp;
+    private String ipAddress;
 
     @SerializedName(ApiConstants.PEERS)
     @Param(description = "the Management Server Peers")
@@ -130,8 +130,8 @@ public Date getLastBoot() {
         return lastBoot;
     }
 
-    public String getServiceIp() {
-        return serviceIp;
+    public String getIpAddress() {
+        return ipAddress;
     }
 
     public Long getAgentsCount() {
@@ -186,8 +186,8 @@ public void setKernelVersion(String kernelVersion) {
         this.kernelVersion = kernelVersion;
     }
 
-    public void setServiceIp(String serviceIp) {
-        this.serviceIp = serviceIp;
+    public void setIpAddress(String ipAddress) {
+        this.ipAddress = ipAddress;
     }
 
     public void setAgentsCount(Long agentsCount) {

core/src/main/java/com/cloud/agent/api/CleanupVMCommand.java

@@ -0,0 +1,46 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+/**
+ * This command will destroy a leftover VM during the expunge process if it wasn't destroyed before.
+ *
+ */
+public class CleanupVMCommand extends Command {
+    String vmName;
+    boolean executeInSequence;
+
+    public CleanupVMCommand(String vmName) {
+        this(vmName, false);
+    }
+    public CleanupVMCommand(String vmName, boolean executeInSequence) {
+        this.vmName = vmName;
+        this.executeInSequence = executeInSequence;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return executeInSequence;
+    }
+
+    public String getVmName() {
+        return vmName;
+    }
+}

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java

@@ -1807,7 +1807,7 @@ private Pair<VolumeVO, DataStore> recreateVolume(VolumeVO vol, VirtualMachinePro
                     try {
                         volService.grantAccess(volFactory.getVolume(newVol.getId()), host, destPool);
                     } catch (Exception e) {
-                        throw new StorageAccessException(String.format("Unable to grant access to the volume [%s] on host [%s].", newVolToString, host));
+                        throw new StorageAccessException(String.format("Unable to grant access to the volume [%s] on host [%s].", newVolToString, host), e);
                     }
                 }
 
@@ -1847,7 +1847,7 @@ protected void grantVolumeAccessToHostIfNeeded(PrimaryDataStore volumeStore, lon
         try {
             volService.grantAccess(volFactory.getVolume(volumeId), host, volumeStore);
         } catch (Exception e) {
-            throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host));
+            throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host), e);
         }
     }
 
@@ -1928,7 +1928,7 @@ public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws Sto
                         try {
                             volService.grantAccess(volFactory.getVolume(vol.getId()), host, store);
                         } catch (Exception e) {
-                            throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host));
+                            throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host), e);
                         }
                     } else {
                         grantVolumeAccessToHostIfNeeded(store, vol.getId(), host, volToString);

engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java

@@ -20,8 +20,9 @@
 import java.util.Map;
 
 import com.cloud.utils.db.GenericDao;
+import org.apache.cloudstack.resourcedetail.ResourceDetailsDao;
 
-public interface ClusterDetailsDao extends GenericDao<ClusterDetailsVO, Long> {
+public interface ClusterDetailsDao extends GenericDao<ClusterDetailsVO, Long>, ResourceDetailsDao<ClusterDetailsVO> {
     Map<String, String> findDetails(long clusterId);
 
     void persist(long clusterId, Map<String, String> details);