From e7ff4edc5697c6d94db9c427e84934899060fae3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 28 Jan 2025 14:37:24 +0530 Subject: [PATCH 01/32] wip Signed-off-by: Abhishek Kumar --- ui/src/config/section/infra/clusters.js | 2 +- ui/src/config/section/infra/hosts.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/config/section/infra/clusters.js b/ui/src/config/section/infra/clusters.js index c7edc6560ef2..b3af174876e3 100644 --- a/ui/src/config/section/infra/clusters.js +++ b/ui/src/config/section/infra/clusters.js @@ -26,7 +26,7 @@ export default { permission: ['listClustersMetrics'], searchFilters: ['name', 'zoneid', 'podid', 'hypervisor'], columns: () => { - const fields = ['name', 'state', 'allocationstate', 'clustertype', 'hypervisortype', 'hosts'] + const fields = ['name', 'state', 'allocationstate', 'clustertype', 'arch', 'hypervisortype', 'hosts'] const metricsFields = ['cpuused', 'cpumaxdeviation', 'cpuallocated', 'cputotal', 'memoryused', 'memorymaxdeviation', 'memoryallocated', 'memorytotal', 'drsimbalance'] if (store.getters.metrics) { fields.push(...metricsFields) diff --git a/ui/src/config/section/infra/hosts.js b/ui/src/config/section/infra/hosts.js index 5def7f3b7fc1..5089715f9692 100644 --- a/ui/src/config/section/infra/hosts.js +++ b/ui/src/config/section/infra/hosts.js @@ -32,7 +32,7 @@ export default { }, params: { type: 'routing' }, columns: () => { - const fields = ['name', 'state', 'resourcestate', 'ipaddress', 'hypervisor', 'instances', 'powerstate', 'version'] + const fields = ['name', 'state', 'resourcestate', 'ipaddress', 'arch', 'hypervisor', 'instances', 'powerstate', 'version'] const metricsFields = ['cpunumber', 'cputotalghz', 'cpuusedghz', 'cpuallocatedghz', 'memorytotalgb', 'memoryusedgb', 'memoryallocatedgb', 'networkread', 'networkwrite'] if (store.getters.metrics) { fields.push(...metricsFields) From 67042ce3497163c04511ae72729f87a854009c22 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 28 Jan 2025 17:04:08 +0530 Subject: [PATCH 02/32] ui: multi-arch improvements Signed-off-by: Abhishek Kumar --- ui/src/config/section/image.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 6001480513bb..1beab17e547c 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -43,7 +43,7 @@ export default { } return 'Not Ready' } - }, 'ostypename', 'hypervisor'] + }, 'ostypename', 'arch', 'hypervisor'] if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { fields.push('size') fields.push('account') @@ -220,7 +220,7 @@ export default { } return 'Not Ready' } - }, 'ostypename'] + }, 'ostypename', 'arch'] if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { fields.push('size') fields.push('account') From 126dfa975b37dba9ad8fcf4da0054043405e6830 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2025 12:01:03 +0530 Subject: [PATCH 03/32] api,ui: filter resources by arch Signed-off-by: Abhishek Kumar --- .../api/command/admin/cluster/ListClustersCmd.java | 9 +++++++++ .../cloudstack/api/command/admin/host/ListHostsCmd.java | 7 +++++++ .../main/java/com/cloud/api/query/QueryManagerImpl.java | 6 ++++++ .../main/java/com/cloud/server/ManagementServerImpl.java | 6 ++++++ ui/src/config/section/image.js | 4 ++-- ui/src/config/section/infra/clusters.js | 2 +- ui/src/config/section/infra/hosts.js | 2 +- 7 files changed, 32 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java index 67d0678410cf..fbe7f517066d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java @@ -68,6 +68,11 @@ public class ListClustersCmd extends BaseListCmd { @Parameter(name = ApiConstants.SHOW_CAPACITIES, type = CommandType.BOOLEAN, description = "flag to display the capacity of the clusters") private Boolean showCapacities; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, + description = "CPU arch of the clusters", + since = "4.20.1") + private String arch; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -112,6 +117,10 @@ public Boolean getShowCapacities() { return showCapacities; } + public String getArch() { + return arch; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index af87bbf33bb0..6c501650585b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -105,6 +105,9 @@ public class ListHostsCmd extends BaseListCmd { @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "hypervisor type of host: XenServer,KVM,VMware,Hyperv,BareMetal,Simulator") private String hypervisor; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, description = "CPU Arch of the host", since = "4.20.1") + private String arch; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -189,6 +192,10 @@ public String getHostOutOfBandManagementPowerState() { return outOfBandManagementPowerState; } + public String getArch() { + return arch; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 26c5cd4e10f1..19b812de384f 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -2346,6 +2346,7 @@ public Pair, Integer> searchForServerIdsAndCount(ListHostsCmd cmd) { Long startIndex = cmd.getStartIndex(); Long pageSize = cmd.getPageSizeVal(); Hypervisor.HypervisorType hypervisorType = cmd.getHypervisor(); + String arch = cmd.getArch(); Filter searchFilter = new Filter(HostVO.class, "id", Boolean.TRUE, startIndex, pageSize); @@ -2361,6 +2362,7 @@ public Pair, Integer> searchForServerIdsAndCount(ListHostsCmd cmd) { hostSearchBuilder.and("clusterId", hostSearchBuilder.entity().getClusterId(), SearchCriteria.Op.EQ); hostSearchBuilder.and("resourceState", hostSearchBuilder.entity().getResourceState(), SearchCriteria.Op.EQ); hostSearchBuilder.and("hypervisor_type", hostSearchBuilder.entity().getHypervisorType(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("arch", hostSearchBuilder.entity().getArch(), SearchCriteria.Op.EQ); if (keyword != null) { hostSearchBuilder.and().op("keywordName", hostSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE); @@ -2441,6 +2443,10 @@ public Pair, Integer> searchForServerIdsAndCount(ListHostsCmd cmd) { sc.setParameters("hypervisor_type", hypervisorType); } + if (StringUtils.isNotBlank(arch)) { + sc.setParameters("arch", arch); + } + Pair, Integer> uniqueHostPair = hostDao.searchAndCount(sc, searchFilter); Integer count = uniqueHostPair.second(); List hostIds = uniqueHostPair.first().stream().map(HostVO::getId).collect(Collectors.toList()); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 8575a26a5cbe..5ec8172e97f2 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1274,6 +1274,7 @@ public Pair, Integer> searchForClusters(final ListCluste final Object clusterType = cmd.getClusterType(); final Object allocationState = cmd.getAllocationState(); final String keyword = cmd.getKeyword(); + final String arch = cmd.getArch(); zoneId = _accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), zoneId); final Filter searchFilter = new Filter(ClusterVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); @@ -1286,6 +1287,7 @@ public Pair, Integer> searchForClusters(final ListCluste sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); sb.and("clusterType", sb.entity().getClusterType(), SearchCriteria.Op.EQ); sb.and("allocationState", sb.entity().getAllocationState(), SearchCriteria.Op.EQ); + sb.and("arch", sb.entity().getArch(), SearchCriteria.Op.EQ); final SearchCriteria sc = sb.create(); if (id != null) { @@ -1325,6 +1327,10 @@ public Pair, Integer> searchForClusters(final ListCluste sc.addAnd("name", SearchCriteria.Op.SC, ssc); } + if (StringUtils.isNotBlank(arch)) { + sc.setParameters("arch", arch); + } + final Pair, Integer> result = _clusterDao.searchAndCount(sc, searchFilter); return new Pair<>(result.first(), result.second()); } diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 1beab17e547c..9870866c4a42 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -67,7 +67,7 @@ export default { return fields }, searchFilters: () => { - var filters = ['name', 'zoneid', 'tags'] + var filters = ['name', 'zoneid', 'tags', 'arch'] if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { filters.push('storageid') filters.push('imagestoreid') @@ -235,7 +235,7 @@ export default { }, details: ['name', 'id', 'displaytext', 'checksum', 'ostypename', 'size', 'arch', 'bootable', 'isready', 'passwordenabled', 'directdownload', 'isextractable', 'ispublic', 'isfeatured', 'isdynamicallyscalable', 'crosszones', 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'url'], searchFilters: () => { - var filters = ['name', 'zoneid', 'tags'] + var filters = ['name', 'zoneid', 'tags', 'arch'] if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { filters.push('storageid') filters.push('imagestoreid') diff --git a/ui/src/config/section/infra/clusters.js b/ui/src/config/section/infra/clusters.js index b3af174876e3..f581505e7cfd 100644 --- a/ui/src/config/section/infra/clusters.js +++ b/ui/src/config/section/infra/clusters.js @@ -24,7 +24,7 @@ export default { icon: 'cluster-outlined', docHelp: 'conceptsandterminology/concepts.html#about-clusters', permission: ['listClustersMetrics'], - searchFilters: ['name', 'zoneid', 'podid', 'hypervisor'], + searchFilters: ['name', 'zoneid', 'podid', 'arch', 'hypervisor'], columns: () => { const fields = ['name', 'state', 'allocationstate', 'clustertype', 'arch', 'hypervisortype', 'hosts'] const metricsFields = ['cpuused', 'cpumaxdeviation', 'cpuallocated', 'cputotal', 'memoryused', 'memorymaxdeviation', 'memoryallocated', 'memorytotal', 'drsimbalance'] diff --git a/ui/src/config/section/infra/hosts.js b/ui/src/config/section/infra/hosts.js index 5089715f9692..0493d942901c 100644 --- a/ui/src/config/section/infra/hosts.js +++ b/ui/src/config/section/infra/hosts.js @@ -24,7 +24,7 @@ export default { icon: 'database-outlined', docHelp: 'conceptsandterminology/concepts.html#about-hosts', permission: ['listHostsMetrics'], - searchFilters: ['name', 'zoneid', 'podid', 'clusterid', 'hypervisor'], + searchFilters: ['name', 'zoneid', 'podid', 'clusterid', 'arch', 'hypervisor'], resourceType: 'Host', filters: () => { const filters = ['enabled', 'disabled', 'maintenance', 'up', 'down', 'disconnected', 'alert'] From 8f48da31bc15b81f48934395d16beb6b9af8e0ff Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2025 12:04:30 +0530 Subject: [PATCH 04/32] ui: show arch tag in infocard Signed-off-by: Abhishek Kumar --- ui/src/components/view/InfoCard.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ui/src/components/view/InfoCard.vue b/ui/src/components/view/InfoCard.vue index e19456ecd5c9..8efed29ba494 100644 --- a/ui/src/components/view/InfoCard.vue +++ b/ui/src/components/view/InfoCard.vue @@ -72,6 +72,9 @@ {{ resource.broadcasturi }} + + {{ resource.arch }} + {{ resource.hypervisor }} From 13f3f148a295f06d25e42260430353cc1bf15b20 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2025 13:07:34 +0530 Subject: [PATCH 05/32] ui: add arch display and filter for kubernetes iso Signed-off-by: Abhishek Kumar --- ui/src/config/section/image.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 9870866c4a42..87766de05306 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -370,8 +370,8 @@ export default { icon: ['fa-solid', 'fa-dharmachakra'], docHelp: 'plugins/cloudstack-kubernetes-service.html#kubernetes-supported-versions', permission: ['listKubernetesSupportedVersions'], - searchFilters: ['zoneid', 'minimumsemanticversion'], - columns: ['name', 'state', 'semanticversion', 'isostate', 'mincpunumber', 'minmemory', 'zonename'], + searchFilters: ['zoneid', 'minimumsemanticversion', 'arch'], + columns: ['name', 'state', 'semanticversion', 'isostate', 'mincpunumber', 'minmemory', 'arch', 'zonename'], details: ['name', 'semanticversion', 'supportsautoscaling', 'zoneid', 'zonename', 'isoid', 'isoname', 'isostate', 'mincpunumber', 'minmemory', 'supportsha', 'state', 'created'], tabs: [ { From b017025f1c4593007af311a9ad90aefe4355cfc1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2025 13:08:10 +0530 Subject: [PATCH 06/32] simulator: allow adding hosts with arch Signed-off-by: Abhishek Kumar --- .../com/cloud/agent/manager/MockAgentManager.java | 1 + .../com/cloud/agent/manager/MockAgentManagerImpl.java | 2 ++ .../java/com/cloud/resource/AgentRoutingResource.java | 3 +++ .../java/com/cloud/resource/SimulatorDiscoverer.java | 4 ++++ .../src/main/java/com/cloud/simulator/MockHost.java | 2 ++ .../src/main/java/com/cloud/simulator/MockHostVO.java | 11 +++++++++++ setup/db/create-schema-simulator.sql | 1 + 7 files changed, 24 insertions(+) diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockAgentManager.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockAgentManager.java index 6a9e70763a21..260ffe594f3c 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockAgentManager.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockAgentManager.java @@ -39,6 +39,7 @@ public interface MockAgentManager extends Manager { public static final long DEFAULT_HOST_MEM_SIZE = 8 * 1024 * 1024 * 1024L; // 8G, unit of Mbytes public static final int DEFAULT_HOST_CPU_CORES = 4; // 2 dual core CPUs (2 x 2) public static final int DEFAULT_HOST_SPEED_MHZ = 8000; // 1 GHz CPUs + public static final String DEFAULT_HOST_ARCH = "x86_64"; @Override boolean configure(String name, Map params) throws ConfigurationException; diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockAgentManagerImpl.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockAgentManagerImpl.java index 8cb2c32a1c89..d3d6f646a7e7 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockAgentManagerImpl.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/agent/manager/MockAgentManagerImpl.java @@ -153,6 +153,7 @@ public Map> createServerResources(Map> createServerResources(Map hostDetails = new HashMap(); hostDetails.put(RouterPrivateIpStrategy.class.getCanonicalName(), RouterPrivateIpStrategy.DcGlobal.toString()); @@ -274,12 +275,14 @@ protected List getHostInfo() { long cpus = agentHost.getCpuCount(); long ram = agentHost.getMemorySize(); long dom0Ram = agentHost.getMemorySize() / 10; + String arch = agentHost.getArch(); info.add((int)cpus); info.add(speed); info.add(ram); info.add(agentHost.getCapabilities()); info.add(dom0Ram); + info.add(arch); return info; } diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorDiscoverer.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorDiscoverer.java index 37b1ca3e4a80..332ac6098033 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorDiscoverer.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorDiscoverer.java @@ -87,6 +87,7 @@ public class SimulatorDiscoverer extends DiscovererBase implements Discoverer, L long cpuCores = MockAgentManager.DEFAULT_HOST_CPU_CORES; long memory = MockAgentManager.DEFAULT_HOST_MEM_SIZE; long localstorageSize = MockStorageManager.DEFAULT_HOST_STORAGE_SIZE; + String arch = MockAgentManager.DEFAULT_HOST_ARCH; if (scheme.equals("http")) { if (host == null || !host.startsWith("sim")) { String msg = "uri is not of simulator type so we're not taking care of the discovery for this: " + uri; @@ -111,6 +112,8 @@ public class SimulatorDiscoverer extends DiscovererBase implements Discoverer, L memory = Long.parseLong(parameter[1]); } else if (parameter[0].equalsIgnoreCase("localstorage") && parameter[1] != null) { localstorageSize = Long.parseLong(parameter[1]); + } else if (parameter[0].equalsIgnoreCase("arch") && parameter[1] != null) { + arch = parameter[1]; } } } @@ -168,6 +171,7 @@ public class SimulatorDiscoverer extends DiscovererBase implements Discoverer, L params.put("cpucore", Long.toString(cpuCores)); params.put("memory", Long.toString(memory)); params.put("localstorage", Long.toString(localstorageSize)); + params.put("arch", arch); resources = createAgentResources(params); return resources; diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/MockHost.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/MockHost.java index 04ef9b6e20d9..94a81ea1fae5 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/MockHost.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/MockHost.java @@ -23,6 +23,8 @@ public interface MockHost { public long getMemorySize(); + String getArch(); + public String getCapabilities(); public long getId(); diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/MockHostVO.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/MockHostVO.java index 89064f8234d0..a766ae123883 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/MockHostVO.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/simulator/MockHostVO.java @@ -87,6 +87,9 @@ public class MockHostVO implements MockHost, InternalIdentity { @Column(name = "ram") private long memorySize; + @Column(name = "arch") + private String arch; + @Column(name = "capabilities") private String capabilities; @@ -143,6 +146,14 @@ public void setMemorySize(long memorySize) { this.memorySize = memorySize; } + public String getArch() { + return arch; + } + + public void setArch(String arch) { + this.arch = arch; + } + @Override public String getCapabilities() { return this.capabilities; diff --git a/setup/db/create-schema-simulator.sql b/setup/db/create-schema-simulator.sql index f52faa043d8f..6cb6786311ae 100644 --- a/setup/db/create-schema-simulator.sql +++ b/setup/db/create-schema-simulator.sql @@ -43,6 +43,7 @@ CREATE TABLE `simulator`.`mockhost` ( `cpus` int(10) unsigned, `speed` int(10) unsigned, `ram` bigint unsigned, + `arch` varchar(8) DEFAULT "x86_64" COMMENT "the CPU architecture of the host", `capabilities` varchar(255) COMMENT 'host capabilities in comma separated list', `vm_id` bigint unsigned, `resource` varchar(255) DEFAULT NULL COMMENT 'If it is a local resource, this is the class name', From ee59e6115acf3e621248215a7a72b9d17fa73d46 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2025 13:31:03 +0530 Subject: [PATCH 07/32] api,ui: allow filtering vms by arch Signed-off-by: Abhishek Kumar --- .../cloudstack/api/command/user/vm/ListVMsCmd.java | 9 +++++++++ .../cloudstack/api/response/UserVmResponse.java | 14 +++++++++++++- .../META-INF/db/views/cloud.user_vm_view.sql | 1 + .../java/com/cloud/api/query/QueryManagerImpl.java | 8 +++++++- .../com/cloud/api/query/dao/UserVmJoinDaoImpl.java | 1 + .../java/com/cloud/api/query/vo/UserVmJoinVO.java | 7 +++++++ ui/src/config/section/compute.js | 2 +- 7 files changed, 39 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java index 50e1798112d2..181a8c5495ff 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java @@ -149,6 +149,11 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements @Parameter(name = ApiConstants.USER_DATA, type = CommandType.BOOLEAN, description = "Whether to return the VMs' user data or not. By default, user data will not be returned.", since = "4.18.0.0") private Boolean showUserData; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, + description = "CPU arch of the VM", + since = "4.20.1") + private String arch; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -284,6 +289,10 @@ public Boolean getVnf() { return isVnf; } + public String getArch() { + return arch; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index 1f4b493fba2f..2f519ccf3a5c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -31,13 +31,13 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponseWithTagInformation; import org.apache.cloudstack.api.EntityReference; +import org.apache.commons.collections.CollectionUtils; import com.cloud.network.router.VirtualRouter; import com.cloud.serializer.Param; import com.cloud.uservm.UserVm; import com.cloud.vm.VirtualMachine; import com.google.gson.annotations.SerializedName; -import org.apache.commons.collections.CollectionUtils; @SuppressWarnings("unused") @EntityReference(value = {VirtualMachine.class, UserVm.class, VirtualRouter.class}) @@ -396,6 +396,10 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co @Param(description = "User VM type", since = "4.20.0") private String vmType; + @SerializedName(ApiConstants.ARCH) + @Param(description = "CPU arch of the VM", since = "4.20.1") + private String arch; + public UserVmResponse() { securityGroupList = new LinkedHashSet<>(); nics = new TreeSet<>(Comparator.comparingInt(x -> Integer.parseInt(x.getDeviceId()))); @@ -1169,4 +1173,12 @@ public String getVmType() { public void setIpAddress(String ipAddress) { this.ipAddress = ipAddress; } + + public String getArch() { + return arch; + } + + public void setArch(String arch) { + this.arch = arch; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index 97cb7b735cfc..430031b1d975 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -83,6 +83,7 @@ SELECT `iso`.`uuid` AS `iso_uuid`, `iso`.`name` AS `iso_name`, `iso`.`display_text` AS `iso_display_text`, + `vm_template`.`arch` AS `arch`, `service_offering`.`id` AS `service_offering_id`, `service_offering`.`uuid` AS `service_offering_uuid`, `disk_offering`.`uuid` AS `disk_offering_uuid`, diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 19b812de384f..ce88e616e762 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -1297,6 +1297,7 @@ private Pair, Integer> searchForUserVMIdsAndCount(ListVMsCmd cmd) { StoragePoolVO pool = null; Long userId = cmd.getUserId(); Map tags = cmd.getTags(); + final String arch = cmd.getArch(); boolean isAdmin = false; boolean isRootAdmin = false; @@ -1520,8 +1521,10 @@ private Pair, Integer> searchForUserVMIdsAndCount(ListVMsCmd cmd) { } Boolean isVnf = cmd.getVnf(); - if (isVnf != null) { + boolean templateJoinNeeded = isVnf != null || StringUtils.isNoneBlank(arch); + if (templateJoinNeeded) { SearchBuilder templateSearch = _templateDao.createSearchBuilder(); + templateSearch.and("templateArch", templateSearch.entity().getArch(), Op.EQ); templateSearch.and("templateTypeEQ", templateSearch.entity().getTemplateType(), Op.EQ); templateSearch.and("templateTypeNEQ", templateSearch.entity().getTemplateType(), Op.NEQ); @@ -1646,6 +1649,9 @@ private Pair, Integer> searchForUserVMIdsAndCount(ListVMsCmd cmd) { userVmSearchCriteria.setJoinParameters("vmTemplate", "templateTypeNEQ", TemplateType.VNF); } } + if (StringUtils.isNotBlank(arch)) { + userVmSearchCriteria.setJoinParameters("vmTemplate", "templateArch", arch); + } if (isRootAdmin) { if (podId != null) { diff --git a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java index 7e10df24e1b5..f5d9fe16925e 100644 --- a/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/UserVmJoinDaoImpl.java @@ -188,6 +188,7 @@ public UserVmResponse newUserVmResponse(ResponseView view, String objectName, Us userVmResponse.setInstanceName(userVm.getInstanceName()); userVmResponse.setHostId(userVm.getHostUuid()); userVmResponse.setHostName(userVm.getHostName()); + userVmResponse.setArch(userVm.getArch()); } if (userVm.getHostStatus() != null) { userVmResponse.setHostControlState(ControlState.getControlState(userVm.getHostStatus(), userVm.getHostResourceState()).toString()); diff --git a/server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java index 701fa7d4f826..90d4a14e93e4 100644 --- a/server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/UserVmJoinVO.java @@ -439,6 +439,9 @@ public class UserVmJoinVO extends BaseViewWithTagInformationVO implements Contro @Column(name = "delete_protection") protected Boolean deleteProtection; + @Column(name = "arch") + protected String arch; + public UserVmJoinVO() { // Empty constructor @@ -977,4 +980,8 @@ public String getUserDataPolicy() { public String getUserDataDetails() { return userDataDetails; } + + public String getArch() { + return arch; + } } diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 55aeb19317b3..c71aa7f4b832 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -78,7 +78,7 @@ export default { fields.push('zonename') return fields }, - searchFilters: ['name', 'zoneid', 'domainid', 'account', 'groupid', 'tags'], + searchFilters: ['name', 'zoneid', 'domainid', 'account', 'groupid', 'arch', 'tags'], details: () => { var fields = ['name', 'displayname', 'id', 'state', 'ipaddress', 'ip6address', 'templatename', 'ostypename', 'serviceofferingname', 'isdynamicallyscalable', 'haenable', 'hypervisor', 'boottype', 'bootmode', 'account', From 13ced5ec4f6802be1945dfa14896615859a6393b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2025 13:31:31 +0530 Subject: [PATCH 08/32] ui: make arch filter a list Signed-off-by: Abhishek Kumar --- ui/src/components/view/SearchView.vue | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ui/src/components/view/SearchView.vue b/ui/src/components/view/SearchView.vue index 912c4d1be7b6..6738c8d5c755 100644 --- a/ui/src/components/view/SearchView.vue +++ b/ui/src/components/view/SearchView.vue @@ -310,7 +310,9 @@ export default { } if (['zoneid', 'domainid', 'imagestoreid', 'storageid', 'state', 'account', 'hypervisor', 'level', 'clusterid', 'podid', 'groupid', 'entitytype', 'accounttype', 'systemvmtype', 'scope', 'provider', - 'type', 'scope', 'managementserverid', 'serviceofferingid', 'diskofferingid', 'networkid', 'usagetype', 'restartrequired', 'displaynetwork'].includes(item) + 'type', 'scope', 'managementserverid', 'serviceofferingid', + 'diskofferingid', 'networkid', 'usagetype', 'restartrequired', + 'displaynetwork', 'arch'].includes(item) ) { type = 'list' } else if (item === 'tags') { @@ -445,6 +447,13 @@ export default { ] this.fields[apiKeyAccessIndex].loading = false } + + if (arrayField.includes('arch')) { + const typeIndex = this.fields.findIndex(item => item.name === 'arch') + this.fields[typeIndex].loading = true + this.fields[typeIndex].opts = this.fetchArchitectureTypes() + this.fields[typeIndex].loading = false + } }, async fetchDynamicFieldData (arrayField, searchKeyword) { const promises = [] @@ -1315,6 +1324,12 @@ export default { }) }) }, + fetchArchitectureTypes () { + return [ + { id: 'x86_64', name: 'AMD 64 bits (x86_64)' }, + { id: 'aarch64', name: 'ARM 64 bits (aarch64)' } + ] + }, onSearch (value) { this.paramsFilter = {} this.searchQuery = value From 16282dd9497702a008617453de420637c15666a3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 4 Feb 2025 16:00:25 +0530 Subject: [PATCH 09/32] wip Signed-off-by: Abhishek Kumar --- engine/schema/pom.xml | 6 + .../java/com/cloud/dc/dao/ClusterDao.java | 3 + .../java/com/cloud/dc/dao/ClusterDaoImpl.java | 21 ++ .../cloud/upgrade/DatabaseUpgradeChecker.java | 13 + .../upgrade/SystemVmTemplateRegistration.java | 335 +++++++++++------- .../storage/datastore/db/ImageStoreDao.java | 2 +- .../datastore/db/ImageStoreDaoImpl.java | 18 +- .../src/main/resources/config.properties | 1 + .../com/cloud/storage/StorageManagerImpl.java | 31 +- 9 files changed, 280 insertions(+), 150 deletions(-) create mode 100644 engine/schema/src/main/resources/config.properties diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 431c2fd2b03c..caebdc9b412d 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -166,6 +166,12 @@ + + + src/main/resources + true + + diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java index bf12abd5114c..ddf503cb1f1c 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java @@ -23,6 +23,7 @@ import com.cloud.cpu.CPU; import com.cloud.dc.ClusterVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; public interface ClusterDao extends GenericDao { @@ -38,6 +39,8 @@ public interface ClusterDao extends GenericDao { Set getDistinctAvailableHypervisorsAcrossClusters(); + List> getDistinctHypervisorsArchAcrossClusters(); + List listByDcHyType(long dcId, String hyType); Map> getPodClusterIdMap(List clusterIds); diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index af6b83976430..f5a2547c2856 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -39,6 +39,7 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.org.Grouping; import com.cloud.org.Managed; +import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; @@ -182,6 +183,26 @@ public Set getDistinctAvailableHypervisorsAcrossClusters() { return new HashSet<>(getAvailableHypervisorInZone(null)); } + @Override + public List> getDistinctHypervisorsArchAcrossClusters() { + List> hypervisorArchList = new ArrayList<>(); + String selectSql = "SELECT DISTINCT hypervisor_type, arch FROM cloud.cluster WHERE removed IS NULL"; + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try { + PreparedStatement stmt = txn.prepareAutoCloseStatement(selectSql); + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + HypervisorType hypervisorType = HypervisorType.valueOf(rs.getString("hypervisor_type")); + String arch = rs.getString("arch"); + hypervisorArchList.add(new Pair<>(hypervisorType, arch)); + } + } catch (SQLException ex) { + logger.error("DB exception {}", ex.getMessage(), ex); + return null; + } + return hypervisorArchList; + } + @Override public Map> getPodClusterIdMap(List clusterIds) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index abf860439375..5edbbdfed11e 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Properties; import javax.inject.Inject; @@ -443,6 +444,18 @@ protected void executeViewScripts() { public void check() { GlobalLock lock = GlobalLock.getInternLock("DatabaseUpgrade"); try { + + + Properties properties = new Properties(); + try (InputStream input = this.getClass().getClassLoader().getResourceAsStream("config.properties")) { + if (input != null) { + properties.load(input); + LOGGER.info("----------------------{}", properties.getProperty("systemvm.template.location", "unknown")); + } + } catch (IOException e) { + LOGGER.error("----------Error reading property", e); + } + LOGGER.info("Grabbing lock to check for database upgrade."); if (!lock.lock(20 * 60)) { throw new CloudRuntimeException("Unable to acquire lock to check for database integrity."); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 9b0e10cbbfcc..ee57bc616a1a 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -16,6 +16,47 @@ // under the License. package com.cloud.upgrade; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Date; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationDaoImpl; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDaoImpl; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDaoImpl; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.ini4j.Ini; + +import com.cloud.cpu.CPU; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.ClusterDaoImpl; @@ -46,44 +87,6 @@ import com.cloud.utils.script.Script; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.dao.VMInstanceDaoImpl; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.dao.ConfigurationDaoImpl; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDaoImpl; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDaoImpl; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.ini4j.Ini; - -import javax.inject.Inject; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.Connection; -import java.sql.Date; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; public class SystemVmTemplateRegistration { protected static Logger LOGGER = LogManager.getLogger(SystemVmTemplateRegistration.class); @@ -288,18 +291,17 @@ public void setUpdated(Date updated) { } } - public static final List hypervisorList = Arrays.asList(Hypervisor.HypervisorType.KVM, - Hypervisor.HypervisorType.VMware, - Hypervisor.HypervisorType.XenServer, - Hypervisor.HypervisorType.Hyperv, - Hypervisor.HypervisorType.LXC, - Hypervisor.HypervisorType.Ovm3 + public static final List> hypervisorList = Arrays.asList( + new Pair<>(Hypervisor.HypervisorType.KVM, CPU.archX86_64Identifier), + new Pair<>(Hypervisor.HypervisorType.KVM, CPU.archARM64Identifier), + new Pair<>(Hypervisor.HypervisorType.VMware, null), + new Pair<>(Hypervisor.HypervisorType.XenServer, null), + new Pair<>(Hypervisor.HypervisorType.Hyperv, null), + new Pair<>(Hypervisor.HypervisorType.LXC, null), + new Pair<>(Hypervisor.HypervisorType.Ovm3, null) ); - public static final Map NewTemplateNameList = new HashMap(); - public static final Map FileNames = new HashMap(); - public static final Map NewTemplateUrl = new HashMap(); - public static final Map NewTemplateChecksum = new HashMap(); + public static final Map NewTemplateMap = new HashMap<>(); public static final Map RouterTemplateConfigurationNames = new HashMap() { { @@ -368,8 +370,19 @@ public boolean validateIfSeeded(TemplateDataStoreVO templDataStoreVO, String url } } - public Long getRegisteredTemplateId(Pair hypervisorAndTemplateName) { - VMTemplateVO vmTemplate = vmTemplateDao.findLatestTemplateByName(hypervisorAndTemplateName.second()); + private static String getHypervisorArchKey(String hypervisorType, String arch) { + if (Hypervisor.HypervisorType.KVM.name().equals(hypervisorType)) { + return String.format("%s-%s", hypervisorType, StringUtils.isBlank(arch) ? CPU.archX86_64Identifier : arch); + } + return hypervisorType; + } + + private static MetadataTemplateDetails getMetadataTemplateDetails(Hypervisor.HypervisorType hypervisorType, String arch) { + return NewTemplateMap.get(getHypervisorArchKey(hypervisorType.name(), arch)); + } + + public Long getRegisteredTemplateId(String templateName) { + VMTemplateVO vmTemplate = vmTemplateDao.findLatestTemplateByName(templateName); Long templateId = null; if (vmTemplate != null) { templateId = vmTemplate.getId(); @@ -379,7 +392,7 @@ public Long getRegisteredTemplateId(Pair hype private static String fetchTemplatesPath() { String filePath = RELATIVE_TEMPLATE_PATH + METADATA_FILE_NAME; - LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath)); + LOGGER.debug("Looking for file [ {} ] in the classpath.", filePath); File metaFile = new File(filePath); String templatePath = null; if (metaFile.exists()) { @@ -389,9 +402,9 @@ private static String fetchTemplatesPath() { filePath = ABSOLUTE_TEMPLATE_PATH + METADATA_FILE_NAME; metaFile = new File(filePath); templatePath = ABSOLUTE_TEMPLATE_PATH; - LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath)); + LOGGER.debug("Looking for file [ {} ] in the classpath.", filePath); if (!metaFile.exists()) { - String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString()); + String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath); LOGGER.error(errMsg); throw new CloudRuntimeException(errMsg); } @@ -460,9 +473,8 @@ public static void mountStore(String storeUrl, String path, String nfsVersion) { } private List fetchAllHypervisors(Long zoneId) { - List hypervisorList = new ArrayList<>(); List hypervisorTypes = clusterDao.getAvailableHypervisorInZone(zoneId); - hypervisorList = hypervisorTypes.stream().distinct().map(Hypervisor.HypervisorType::name).collect(Collectors.toList()); + List hypervisorList = hypervisorTypes.stream().distinct().map(Hypervisor.HypervisorType::name).collect(Collectors.toList()); return hypervisorList; } @@ -627,16 +639,16 @@ public static void unmountStore(String filePath) { } } - private void setupTemplate(String templateName, Pair hypervisorAndTemplateName, - String destTempFolder) throws CloudRuntimeException { + private void setupTemplate(String templateName, Hypervisor.HypervisorType hypervisor, String destTempFolder) + throws CloudRuntimeException { String setupTmpltScript = Script.findScript(storageScriptsDir, "setup-sysvm-tmplt"); if (setupTmpltScript == null) { throw new CloudRuntimeException("Unable to find the createtmplt.sh"); } Script scr = new Script(setupTmpltScript, SCRIPT_TIMEOUT, LOGGER); scr.add("-u", templateName); - scr.add("-f", TEMPLATES_PATH + FileNames.get(hypervisorAndTemplateName.first())); - scr.add("-h", hypervisorAndTemplateName.first().name().toLowerCase(Locale.ROOT)); + scr.add("-f", TEMPLATES_PATH + NewTemplateMap.get(hypervisor).getFilename()); + scr.add("-h", hypervisor.name().toLowerCase(Locale.ROOT)); scr.add("-d", destTempFolder); String result = scr.execute(); if (result != null) { @@ -647,13 +659,12 @@ private void setupTemplate(String templateName, Pair hypervisorAndTemplateName, - String url, String checksum, ImageFormat format, long guestOsId, - Long storeId, Long templateId, String filePath, TemplateDataStoreVO templateDataStoreVO) { - Hypervisor.HypervisorType hypervisor = hypervisorAndTemplateName.first(); + private Long performTemplateRegistrationOperations(Hypervisor.HypervisorType hypervisor, + String name, CPU.CPUArch arch, String url, String checksum, ImageFormat format, long guestOsId, + Long storeId, Long templateId, String filePath, TemplateDataStoreVO templateDataStoreVO) { String templateName = UUID.randomUUID().toString(); Date created = new Date(DateUtil.currentGMTTime().getTime()); - SystemVMTemplateDetails details = new SystemVMTemplateDetails(templateName, hypervisorAndTemplateName.second(), created, + SystemVMTemplateDetails details = new SystemVMTemplateDetails(templateName, name, created, url, checksum, format, (int) guestOsId, hypervisor, storeId); if (templateId == null) { VMTemplateVO template = createTemplateObjectInDB(details); @@ -671,23 +682,23 @@ private Long performTemplateRegistrationOperations(Pair hypervisorAndTemplateName, - Pair storeUrlAndId, VMTemplateVO templateVO, - TemplateDataStoreVO templateDataStoreVO, String filePath) { + public void registerTemplate(Hypervisor.HypervisorType hypervisor, String name, Pair storeUrlAndId, + VMTemplateVO templateVO, TemplateDataStoreVO templateDataStoreVO, String filePath) { Long templateId = null; try { templateId = templateVO.getId(); - performTemplateRegistrationOperations(hypervisorAndTemplateName, templateVO.getUrl(), templateVO.getChecksum(), - templateVO.getFormat(), templateVO.getGuestOSId(), storeUrlAndId.second(), templateId, filePath, templateDataStoreVO); + performTemplateRegistrationOperations(hypervisor, name, templateVO.getArch(), templateVO.getUrl(), + templateVO.getChecksum(), templateVO.getFormat(), templateVO.getGuestOSId(), storeUrlAndId.second(), + templateId, filePath, templateDataStoreVO); } catch (Exception e) { - String errMsg = String.format("Failed to register template for hypervisor: %s", hypervisorAndTemplateName.first()); + String errMsg = String.format("Failed to register template for hypervisor: %s", hypervisor); LOGGER.error(errMsg, e); if (templateId != null) { updateTemplateTablesOnFailure(templateId); @@ -697,19 +708,22 @@ public void registerTemplate(Pair hypervisorA } } - public void registerTemplate(Pair hypervisorAndTemplateName, Pair storeUrlAndId, String filePath) { + public void registerTemplate(Hypervisor.HypervisorType hypervisor, String name, Pair storeUrlAndId, + String filePath) { Long templateId = null; try { - Hypervisor.HypervisorType hypervisor = hypervisorAndTemplateName.first(); - templateId = performTemplateRegistrationOperations(hypervisorAndTemplateName, NewTemplateUrl.get(hypervisor), NewTemplateChecksum.get(hypervisor), - hypervisorImageFormat.get(hypervisor), hypervisorGuestOsMap.get(hypervisor), storeUrlAndId.second(), null, filePath, null); + MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisor, null); + templateId = performTemplateRegistrationOperations(hypervisor, name, + CPU.CPUArch.fromType(templateDetails.getArch()), templateDetails.getUrl(), + templateDetails.getChecksum(), hypervisorImageFormat.get(hypervisor), + hypervisorGuestOsMap.get(hypervisor), storeUrlAndId.second(), null, filePath, null); Map configParams = new HashMap<>(); - configParams.put(RouterTemplateConfigurationNames.get(hypervisorAndTemplateName.first()), hypervisorAndTemplateName.second()); + configParams.put(RouterTemplateConfigurationNames.get(hypervisor), templateDetails.getName()); configParams.put("minreq.sysvmtemplate.version", getSystemVmTemplateVersion()); updateConfigurationParams(configParams); - updateSystemVMEntries(templateId, hypervisorAndTemplateName.first()); + updateSystemVMEntries(templateId, hypervisor); } catch (Exception e) { - String errMsg = String.format("Failed to register template for hypervisor: %s", hypervisorAndTemplateName.first()); + String errMsg = String.format("Failed to register template for hypervisor: %s", hypervisor); LOGGER.error(errMsg, e); if (templateId != null) { updateTemplateTablesOnFailure(templateId); @@ -729,13 +743,12 @@ public static String parseMetadataFile() { try { Ini ini = new Ini(); ini.load(new FileReader(METADATA_FILE)); - for (Hypervisor.HypervisorType hypervisorType : hypervisorList) { - String hypervisor = hypervisorType.name().toLowerCase(Locale.ROOT); - Ini.Section section = ini.get(hypervisor); - NewTemplateNameList.put(hypervisorType, section.get("templatename")); - FileNames.put(hypervisorType, section.get("filename")); - NewTemplateChecksum.put(hypervisorType, section.get("checksum")); - NewTemplateUrl.put(hypervisorType, section.get("downloadurl")); + for (Pair hypervisorType : hypervisorList) { + String key = getHypervisorArchKey(hypervisorType.first().name(), hypervisorType.second()); + Ini.Section section = ini.get(key); + NewTemplateMap.put(key, new MetadataTemplateDetails(hypervisorType.first(), section.get("templatename"), + section.get("filename"), section.get("downloadurl"), section.get("checksum"), + section.get("arch"))); } Ini.Section section = ini.get("default"); return section.get("version"); @@ -755,12 +768,45 @@ private static void cleanupStore(Long templateId, String filePath) { } } + private void validateTemplates1(Set hypervisorsInUse) { + Set hypervisors = hypervisorsInUse.stream(). + map(Hypervisor.HypervisorType::name).map(name -> name.toLowerCase(Locale.ROOT)).map(this::getHypervisorName).collect(Collectors.toSet()); + List templates = new ArrayList<>(); + for (Hypervisor.HypervisorType hypervisorType : hypervisorsInUse) { + templates.add(NewTemplateMap.get(hypervisorType).getFilename()); + } + + boolean templatesFound = true; + for (String hypervisor : hypervisors) { + String matchedTemplate = templates.stream().filter(x -> x.contains(hypervisor)).findAny().orElse(null); + if (matchedTemplate == null) { + templatesFound = false; + break; + } + + File tempFile = new File(TEMPLATES_PATH + matchedTemplate); + String templateChecksum = DigestHelper.calculateChecksum(tempFile); + String checksumFromMap = NewTemplateMap.get(getHypervisorType(hypervisor)).getChecksum(); + if (!templateChecksum.equals(checksumFromMap)) { + LOGGER.error("Checksum mismatch: {} != {}", templateChecksum, checksumFromMap); + templatesFound = false; + break; + } + } + + if (!templatesFound) { + String errMsg = "SystemVm template not found. Cannot upgrade system Vms"; + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + } + private void validateTemplates(Set hypervisorsInUse) { Set hypervisors = hypervisorsInUse.stream(). map(Hypervisor.HypervisorType::name).map(name -> name.toLowerCase(Locale.ROOT)).map(this::getHypervisorName).collect(Collectors.toSet()); List templates = new ArrayList<>(); for (Hypervisor.HypervisorType hypervisorType : hypervisorsInUse) { - templates.add(FileNames.get(hypervisorType)); + templates.add(NewTemplateMap.get(hypervisorType).getFilename()); } boolean templatesFound = true; @@ -773,8 +819,9 @@ private void validateTemplates(Set hypervisorsInUse) File tempFile = new File(TEMPLATES_PATH + matchedTemplate); String templateChecksum = DigestHelper.calculateChecksum(tempFile); - if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { - LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); + String checksumFromMap = NewTemplateMap.get(getHypervisorType(hypervisor)).getChecksum(); + if (!templateChecksum.equals(checksumFromMap)) { + LOGGER.error("Checksum mismatch: {} != {}", templateChecksum, checksumFromMap); templatesFound = false; break; } @@ -813,10 +860,9 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { mountStore(storeUrlAndId.first(), filePath, nfsVersion); List hypervisorList = fetchAllHypervisors(zoneId); for (String hypervisor : hypervisorList) { - Hypervisor.HypervisorType name = Hypervisor.HypervisorType.getType(hypervisor); - String templateName = NewTemplateNameList.get(name); - Pair hypervisorAndTemplateName = new Pair(name, templateName); - Long templateId = getRegisteredTemplateId(hypervisorAndTemplateName); + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.getType(hypervisor); + MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisorType, null); + Long templateId = getRegisteredTemplateId(templateDetails.getName()); if (templateId != null) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId); @@ -827,12 +873,12 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } } if (templateVO != null) { - registerTemplate(hypervisorAndTemplateName, storeUrlAndId, templateVO, templateDataStoreVO, filePath); - updateRegisteredTemplateDetails(templateId, hypervisorAndTemplateName); + registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId, templateVO, templateDataStoreVO, filePath); + updateRegisteredTemplateDetails(templateId, templateDetails); continue; } } - registerTemplate(hypervisorAndTemplateName, storeUrlAndId, filePath); + registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId, filePath); } unmountStore(filePath); } catch (Exception e) { @@ -851,12 +897,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } } - private void updateRegisteredTemplateDetails(Long templateId, Map.Entry hypervisorAndTemplateName) { - Pair entry = new Pair<>(hypervisorAndTemplateName.getKey(), hypervisorAndTemplateName.getValue()); - updateRegisteredTemplateDetails(templateId, entry); - } - - private void updateRegisteredTemplateDetails(Long templateId, Pair hypervisorAndTemplateName) { + private void updateRegisteredTemplateDetails(Long templateId, MetadataTemplateDetails templateDetails) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); templateVO.setTemplateType(Storage.TemplateType.SYSTEM); boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); @@ -865,22 +906,21 @@ private void updateRegisteredTemplateDetails(Long templateId, Pair configParams = new HashMap<>(); - configParams.put(RouterTemplateConfigurationNames.get(hypervisorAndTemplateName.first()), hypervisorAndTemplateName.second()); + configParams.put(RouterTemplateConfigurationNames.get(hypervisorType), templateDetails.getName()); configParams.put("minreq.sysvmtemplate.version", getSystemVmTemplateVersion()); updateConfigurationParams(configParams); } - private void updateTemplateUrlAndChecksum(VMTemplateVO templateVO, Map.Entry hypervisorAndTemplateName) { - templateVO.setUrl(NewTemplateUrl.get(hypervisorAndTemplateName.getKey())); - templateVO.setChecksum(NewTemplateChecksum.get(hypervisorAndTemplateName.getKey())); + private void updateTemplateUrlAndChecksum(VMTemplateVO templateVO, MetadataTemplateDetails templateDetails) { + templateVO.setUrl(templateDetails.getUrl()); + templateVO.setChecksum(templateDetails.getChecksum()); boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { - String errMsg = String.format("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type %s", hypervisorAndTemplateName.getKey().name()); + String errMsg = String.format("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type %s", templateDetails.getHypervisorType()); LOGGER.error(errMsg); throw new CloudRuntimeException(errMsg); } @@ -891,37 +931,37 @@ public void updateSystemVmTemplates(final Connection conn) { Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { - Set hypervisorsListInUse = new HashSet(); + List> hypervisorsInUse; try { - hypervisorsListInUse = clusterDao.getDistinctAvailableHypervisorsAcrossClusters(); - + hypervisorsInUse = clusterDao.getDistinctHypervisorsArchAcrossClusters(); } catch (final Exception e) { - LOGGER.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: " + e.getMessage()); + LOGGER.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: {}", e.getMessage()); throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting hypervisor types from clusters", e); } - - for (final Map.Entry hypervisorAndTemplateName : NewTemplateNameList.entrySet()) { - LOGGER.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms"); - Long templateId = getRegisteredTemplateId(new Pair<>(hypervisorAndTemplateName.getKey(), hypervisorAndTemplateName.getValue())); + Collection templateEntries = NewTemplateMap.values(); + for (MetadataTemplateDetails templateDetails : templateEntries) { + LOGGER.debug("Updating {} System Vms", templateDetails.getKey()); + Long templateId = getRegisteredTemplateId(templateDetails.getName()); try { // change template type to SYSTEM if (templateId != null) { - updateRegisteredTemplateDetails(templateId, hypervisorAndTemplateName); + updateRegisteredTemplateDetails(templateId, templateDetails); } else { - if (hypervisorsListInUse.contains(hypervisorAndTemplateName.getKey())) { + if (hypervisorsInUse.stream().anyMatch().contains(templateDetails.getHypervisorType())) { try { - registerTemplates(hypervisorsListInUse); + registerTemplates(hypervisorsInUse); break; } catch (final Exception e) { - throw new CloudRuntimeException(String.format("%s %s SystemVm template not found. Cannot upgrade system Vms", getSystemVmTemplateVersion(), hypervisorAndTemplateName.getKey())); + throw new CloudRuntimeException(String.format("%s %s SystemVm template not found. Cannot upgrade system Vms", + getSystemVmTemplateVersion(), templateDetails.getHypervisorType())); } } else { LOGGER.warn(String.format("%s %s SystemVm template not found. Cannot upgrade system Vms hypervisor is not used, so not failing upgrade", - getSystemVmTemplateVersion(), hypervisorAndTemplateName.getKey())); + getSystemVmTemplateVersion(), templateDetails.getHypervisorType())); // Update the latest template URLs for corresponding hypervisor - VMTemplateVO templateVO = vmTemplateDao.findLatestTemplateByTypeAndHypervisor(hypervisorAndTemplateName.getKey(), Storage.TemplateType.SYSTEM); + VMTemplateVO templateVO = vmTemplateDao.findLatestTemplateByTypeAndHypervisor(templateDetails.getHypervisorType(), Storage.TemplateType.SYSTEM); if (templateVO != null) { - updateTemplateUrlAndChecksum(templateVO, hypervisorAndTemplateName); + updateTemplateUrlAndChecksum(templateVO, templateDetails); } } } @@ -948,4 +988,51 @@ public String getNfsVersion(long storeId) { } return null; } + + protected static class MetadataTemplateDetails { + private final Hypervisor.HypervisorType hypervisorType; + private final String name; + private final String filename; + private final String url; + private final String checksum; + private final String arch; + + MetadataTemplateDetails(Hypervisor.HypervisorType hypervisorType, String name, String filename, String url, + String checksum, String arch) { + this.hypervisorType = hypervisorType; + this.name = name; + this.filename = filename; + this.url = url; + this.checksum = checksum; + this.arch = arch; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public String getName() { + return name; + } + + public String getFilename() { + return filename; + } + + public String getUrl() { + return url; + } + + public String getChecksum() { + return checksum; + } + + public String getArch() { + return arch; + } + + public String getKey() { + return getHypervisorArchKey(hypervisorType.name(), arch); + } + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index 7aab5bbf7b31..eda4bcfdaa1f 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -44,7 +44,7 @@ public interface ImageStoreDao extends GenericDao { List listStoresByZoneId(long zoneId); - List listAllStoresInZone(Long zoneId, String provider, DataStoreRole role); + List listAllStoresInZoneExceptId(Long zoneId, String provider, DataStoreRole role, long storeId); List findByProtocol(String protocol); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java index 84b88c215ca6..4cb40b5eaf63 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java @@ -41,7 +41,7 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem private SearchBuilder nameSearch; private SearchBuilder providerSearch; private SearchBuilder regionSearch; - private SearchBuilder storeSearch; + private SearchBuilder storesExceptIdSearch; private SearchBuilder protocolSearch; private SearchBuilder zoneProtocolSearch; @@ -88,11 +88,12 @@ public boolean configure(String name, Map params) throws Configu regionSearch.and("role", regionSearch.entity().getRole(), SearchCriteria.Op.EQ); regionSearch.done(); - storeSearch = createSearchBuilder(); - storeSearch.and("providerName", storeSearch.entity().getProviderName(), SearchCriteria.Op.EQ); - storeSearch.and("role", storeSearch.entity().getRole(), SearchCriteria.Op.EQ); - storeSearch.and("dataCenterId", storeSearch.entity().getDcId(), SearchCriteria.Op.EQ); - storeSearch.done(); + storesExceptIdSearch = createSearchBuilder(); + storesExceptIdSearch.and("providerName", storesExceptIdSearch.entity().getProviderName(), SearchCriteria.Op.EQ); + storesExceptIdSearch.and("role", storesExceptIdSearch.entity().getRole(), SearchCriteria.Op.EQ); + storesExceptIdSearch.and("dataCenterId", storesExceptIdSearch.entity().getDcId(), SearchCriteria.Op.EQ); + storesExceptIdSearch.and("id", storesExceptIdSearch.entity().getId(), SearchCriteria.Op.NEQ); + storesExceptIdSearch.done(); return true; } @@ -113,11 +114,12 @@ public List findByProvider(String provider) { } @Override - public List listAllStoresInZone(Long zoneId, String provider, DataStoreRole role) { - SearchCriteria sc = storeSearch.create(); + public List listAllStoresInZoneExceptId(Long zoneId, String provider, DataStoreRole role, long id) { + SearchCriteria sc = storesExceptIdSearch.create(); sc.setParameters("providerName", provider); sc.setParameters("role", role); sc.setParameters("dataCenterId", zoneId); + sc.setParameters("id", id); return listBy(sc); } diff --git a/engine/schema/src/main/resources/config.properties b/engine/schema/src/main/resources/config.properties new file mode 100644 index 000000000000..6370df704a51 --- /dev/null +++ b/engine/schema/src/main/resources/config.properties @@ -0,0 +1 @@ +systemvm.template.location=${project.systemvm.template.location} \ No newline at end of file diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 74b7f6f358b5..1a598b8e114f 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -3590,12 +3590,11 @@ private void registerSystemVmTemplateOnFirstNfsStore(Long zoneId, String provide Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { - List stores = _imageStoreDao.listAllStoresInZone(zoneId, providerName, DataStoreRole.Image); - stores = stores.stream().filter(str -> str.getId() != store.getId()).collect(Collectors.toList()); - // Check if it's the only/first store in the zone - if (stores.size() == 0) { + List stores = _imageStoreDao.listAllStoresInZoneExceptId(zoneId, providerName, + DataStoreRole.Image, store.getId()); + if (CollectionUtils.isEmpty(stores)) { List hypervisorTypes = _clusterDao.getAvailableHypervisorInZone(zoneId); - Set hypSet = new HashSet<>(hypervisorTypes); + Set hypervisorTypeSet = new HashSet<>(hypervisorTypes); TransactionLegacy txn = TransactionLegacy.open("AutomaticTemplateRegister"); SystemVmTemplateRegistration systemVmTemplateRegistration = new SystemVmTemplateRegistration(); String filePath = null; @@ -3606,24 +3605,22 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } Pair storeUrlAndId = new Pair<>(url, store.getId()); String nfsVersion = imageStoreDetailsUtil.getNfsVersion(store.getId()); - for (HypervisorType hypervisorType : hypSet) { + for (HypervisorType hypervisorType : hypervisorTypeSet) { try { if (HypervisorType.Simulator == hypervisorType) { continue; } String templateName = getValidTemplateName(zoneId, hypervisorType); - Pair hypervisorAndTemplateName = - new Pair<>(hypervisorType, templateName); - Long templateId = systemVmTemplateRegistration.getRegisteredTemplateId(hypervisorAndTemplateName); + Long templateId = systemVmTemplateRegistration.getRegisteredTemplateId(templateName); VMTemplateVO vmTemplateVO = null; - TemplateDataStoreVO templateVO = null; + TemplateDataStoreVO templateDataStoreVO = null; if (templateId != null) { vmTemplateVO = _templateDao.findById(templateId); - templateVO = _templateStoreDao.findByStoreTemplate(store.getId(), templateId); - if (templateVO != null) { + templateDataStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), templateId); + if (templateDataStoreVO != null) { try { - if (systemVmTemplateRegistration.validateIfSeeded( - templateVO, url, templateVO.getInstallPath(), nfsVersion)) { + if (systemVmTemplateRegistration.validateIfSeeded(templateDataStoreVO, + url, templateDataStoreVO.getInstallPath(), nfsVersion)) { continue; } } catch (Exception e) { @@ -3632,10 +3629,10 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } } SystemVmTemplateRegistration.mountStore(storeUrlAndId.first(), filePath, nfsVersion); - if (templateVO != null && vmTemplateVO != null) { - systemVmTemplateRegistration.registerTemplate(hypervisorAndTemplateName, storeUrlAndId, vmTemplateVO, templateVO, filePath); + if (templateDataStoreVO != null && vmTemplateVO != null) { + systemVmTemplateRegistration.registerTemplate(hypervisorType, templateName, storeUrlAndId, vmTemplateVO, templateDataStoreVO, filePath); } else { - systemVmTemplateRegistration.registerTemplate(hypervisorAndTemplateName, storeUrlAndId, filePath); + systemVmTemplateRegistration.registerTemplate(hypervisorType, templateName, storeUrlAndId, filePath); } } catch (CloudRuntimeException e) { SystemVmTemplateRegistration.unmountStore(filePath); From bed9666a4a80a553f25ec17cbb5898e9d43b9175 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 5 Feb 2025 09:54:02 +0530 Subject: [PATCH 10/32] wip Signed-off-by: Abhishek Kumar --- .../upgrade/SystemVmTemplateRegistration.java | 161 ++++++++++-------- 1 file changed, 88 insertions(+), 73 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index ee57bc616a1a..48cada4304ed 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -87,6 +87,47 @@ import com.cloud.utils.script.Script; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.dao.VMInstanceDaoImpl; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationDaoImpl; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDaoImpl; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDaoImpl; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.ini4j.Ini; + +import javax.inject.Inject; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Date; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; public class SystemVmTemplateRegistration { protected static Logger LOGGER = LogManager.getLogger(SystemVmTemplateRegistration.class); @@ -768,32 +809,32 @@ private static void cleanupStore(Long templateId, String filePath) { } } - private void validateTemplates1(Set hypervisorsInUse) { - Set hypervisors = hypervisorsInUse.stream(). - map(Hypervisor.HypervisorType::name).map(name -> name.toLowerCase(Locale.ROOT)).map(this::getHypervisorName).collect(Collectors.toSet()); - List templates = new ArrayList<>(); - for (Hypervisor.HypervisorType hypervisorType : hypervisorsInUse) { - templates.add(NewTemplateMap.get(hypervisorType).getFilename()); - } - + private void validateTemplates(List> hypervisorsArchInUse) { boolean templatesFound = true; - for (String hypervisor : hypervisors) { - String matchedTemplate = templates.stream().filter(x -> x.contains(hypervisor)).findAny().orElse(null); + for (Pair hypervisorArch : hypervisorsArchInUse) { + MetadataTemplateDetails matchedTemplate = NewTemplateMap.values() + .stream() + .filter(x -> x.getHypervisorType().equals(hypervisorArch.first()) && + Objects.equals(x.getArch(), hypervisorArch.second())) + .findAny() + .orElse(null); if (matchedTemplate == null) { templatesFound = false; break; } - - File tempFile = new File(TEMPLATES_PATH + matchedTemplate); + if (CPU.archARM64Identifier.equals(matchedTemplate.getArch())) { + LOGGER.debug("Skipping checksum comparison for the template file and metadata as the arch for template is {}", + matchedTemplate.getArch()); + } + File tempFile = new File(TEMPLATES_PATH + matchedTemplate.getFilename()); String templateChecksum = DigestHelper.calculateChecksum(tempFile); - String checksumFromMap = NewTemplateMap.get(getHypervisorType(hypervisor)).getChecksum(); - if (!templateChecksum.equals(checksumFromMap)) { - LOGGER.error("Checksum mismatch: {} != {}", templateChecksum, checksumFromMap); + if (!templateChecksum.equals(matchedTemplate.getChecksum())) { + LOGGER.error("Checksum {} for file {} does not match checksum {} from metadata", + templateChecksum, matchedTemplate.getChecksum()); templatesFound = false; break; } } - if (!templatesFound) { String errMsg = "SystemVm template not found. Cannot upgrade system Vms"; LOGGER.error(errMsg); @@ -801,40 +842,35 @@ private void validateTemplates1(Set hypervisorsInUse) } } - private void validateTemplates(Set hypervisorsInUse) { - Set hypervisors = hypervisorsInUse.stream(). - map(Hypervisor.HypervisorType::name).map(name -> name.toLowerCase(Locale.ROOT)).map(this::getHypervisorName).collect(Collectors.toSet()); - List templates = new ArrayList<>(); - for (Hypervisor.HypervisorType hypervisorType : hypervisorsInUse) { - templates.add(NewTemplateMap.get(hypervisorType).getFilename()); - } - - boolean templatesFound = true; - for (String hypervisor : hypervisors) { - String matchedTemplate = templates.stream().filter(x -> x.contains(hypervisor)).findAny().orElse(null); - if (matchedTemplate == null) { - templatesFound = false; - break; - } - - File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = DigestHelper.calculateChecksum(tempFile); - String checksumFromMap = NewTemplateMap.get(getHypervisorType(hypervisor)).getChecksum(); - if (!templateChecksum.equals(checksumFromMap)) { - LOGGER.error("Checksum mismatch: {} != {}", templateChecksum, checksumFromMap); - templatesFound = false; - break; + protected void registerTemplatesForZone(long zoneId) { + Pair storeUrlAndId = getNfsStoreInZone(zoneId); + String nfsVersion = getNfsVersion(storeUrlAndId.second()); + mountStore(storeUrlAndId.first(), filePath, nfsVersion); + List hypervisorList = fetchAllHypervisors(zoneId); + for (String hypervisor : hypervisorList) { + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.getType(hypervisor); + MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisorType, null); + Long templateId = getRegisteredTemplateId(templateDetails.getName()); + if (templateId != null) { + VMTemplateVO templateVO = vmTemplateDao.findById(templateId); + TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId); + if (templateDataStoreVO != null) { + String installPath = templateDataStoreVO.getInstallPath(); + if (validateIfSeeded(storeUrlAndId.first(), installPath, nfsVersion)) { + continue; + } + } + if (templateVO != null) { + registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId, templateVO, templateDataStoreVO, filePath); + updateRegisteredTemplateDetails(templateId, templateDetails); + continue; + } } - } - - if (!templatesFound) { - String errMsg = "SystemVm template not found. Cannot upgrade system Vms"; - LOGGER.error(errMsg); - throw new CloudRuntimeException(errMsg); + registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId, filePath); } } - public void registerTemplates(Set hypervisorsInUse) { + public void registerTemplates(List> hypervisorsArchInUse) { GlobalLock lock = GlobalLock.getInternLock("UpgradeDatabase-Lock"); try { LOGGER.info("Grabbing lock to register templates."); @@ -842,7 +878,7 @@ public void registerTemplates(Set hypervisorsInUse) { throw new CloudRuntimeException("Unable to acquire lock to register SystemVM template."); } try { - validateTemplates(hypervisorsInUse); + validateTemplates(hypervisorsArchInUse); // Perform Registration if templates not already registered Transaction.execute(new TransactionCallbackNoReturn() { @Override @@ -855,31 +891,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (filePath == null) { throw new CloudRuntimeException("Failed to create temporary file path to mount the store"); } - Pair storeUrlAndId = getNfsStoreInZone(zoneId); - String nfsVersion = getNfsVersion(storeUrlAndId.second()); - mountStore(storeUrlAndId.first(), filePath, nfsVersion); - List hypervisorList = fetchAllHypervisors(zoneId); - for (String hypervisor : hypervisorList) { - Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.getType(hypervisor); - MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisorType, null); - Long templateId = getRegisteredTemplateId(templateDetails.getName()); - if (templateId != null) { - VMTemplateVO templateVO = vmTemplateDao.findById(templateId); - TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId); - if (templateDataStoreVO != null) { - String installPath = templateDataStoreVO.getInstallPath(); - if (validateIfSeeded(templateDataStoreVO, storeUrlAndId.first(), installPath, nfsVersion)) { - continue; - } - } - if (templateVO != null) { - registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId, templateVO, templateDataStoreVO, filePath); - updateRegisteredTemplateDetails(templateId, templateDetails); - continue; - } - } - registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId, filePath); - } + registerTemplatesForZone(zoneId); unmountStore(filePath); } catch (Exception e) { unmountStore(filePath); @@ -947,7 +959,10 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (templateId != null) { updateRegisteredTemplateDetails(templateId, templateDetails); } else { - if (hypervisorsInUse.stream().anyMatch().contains(templateDetails.getHypervisorType())) { + boolean isHypervisorArchMatchMetadata = hypervisorsInUse.stream() + .anyMatch(p -> p.first().equals(templateDetails.getHypervisorType()) + || Objects.equals(p.second(), templateDetails.getArch())); + if (isHypervisorArchMatchMetadata) { try { registerTemplates(hypervisorsInUse); break; From 969660a177ba8a7621e49c31faf4f9f19b72439c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 20 Feb 2025 19:53:48 +0530 Subject: [PATCH 11/32] change for downloading templates if allowed Signed-off-by: Abhishek Kumar --- .../upgrade/SystemVmTemplateRegistration.java | 83 +++++++------------ engine/schema/templateConfig.sh | 13 +-- .../java/com/cloud/utils/net/NetUtils.java | 53 +++++++++++- 3 files changed, 90 insertions(+), 59 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 48cada4304ed..8eb0204ee0ea 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -33,7 +33,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; +import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; @@ -84,54 +84,14 @@ import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.Script; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.dao.VMInstanceDaoImpl; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.dao.ConfigurationDaoImpl; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDaoImpl; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDaoImpl; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.ini4j.Ini; - -import javax.inject.Inject; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.Connection; -import java.sql.Date; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; public class SystemVmTemplateRegistration { protected static Logger LOGGER = LogManager.getLogger(SystemVmTemplateRegistration.class); - private static final String MOUNT_COMMAND = "sudo mount -t nfs %s %s"; + private static final String MOUNT_COMMAND_BASE = "sudo mount -t nfs"; private static final String UMOUNT_COMMAND = "sudo umount %s"; private static final String RELATIVE_TEMPLATE_PATH = "./engine/schema/dist/systemvm-templates/"; private static final String ABSOLUTE_TEMPLATE_PATH = "/usr/share/cloudstack-management/templates/systemvm/"; @@ -146,6 +106,9 @@ public class SystemVmTemplateRegistration { private static final Integer LINUX_7_ID = 183; private static final Integer SCRIPT_TIMEOUT = 1800000; private static final Integer LOCK_WAIT_TIMEOUT = 1200; + private static final List DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList( + CPU.archARM64Identifier + ); public static String CS_MAJOR_VERSION = null; @@ -193,7 +156,7 @@ public SystemVmTemplateRegistration(String systemVmTemplateVersion) { } public static String getMountCommand(String nfsVersion, String device, String dir) { - String cmd = "sudo mount -t nfs"; + String cmd = MOUNT_COMMAND_BASE; if (StringUtils.isNotBlank(nfsVersion)) { cmd = String.format("%s -o vers=%s", cmd, nfsVersion); } @@ -789,7 +752,7 @@ public static String parseMetadataFile() { Ini.Section section = ini.get(key); NewTemplateMap.put(key, new MetadataTemplateDetails(hypervisorType.first(), section.get("templatename"), section.get("filename"), section.get("downloadurl"), section.get("checksum"), - section.get("arch"))); + hypervisorType.second())); } Ini.Section section = ini.get("default"); return section.get("version"); @@ -809,6 +772,21 @@ private static void cleanupStore(Long templateId, String filePath) { } } + protected File getTemplateFile(MetadataTemplateDetails templateDetails) { + final String filePath = TEMPLATES_PATH + templateDetails.getFilename(); + File tempFile = new File(filePath); + if (!tempFile.exists() && DOWNLOADABLE_TEMPLATE_ARCH_TYPES.contains(templateDetails.getArch()) && + StringUtils.isNotBlank(templateDetails.getUrl())) { + LOGGER.debug("Downloading the template file {} for hypervisor {} and arch {} as it is not present", + templateDetails.getUrl(), templateDetails.getHypervisorType().name(), templateDetails.getArch()); + if (!NetUtils.downloadFileWithProgress(templateDetails.getUrl(), filePath, LOGGER)) { + return null; + } + return new File(filePath); + } + return tempFile; + } + private void validateTemplates(List> hypervisorsArchInUse) { boolean templatesFound = true; for (Pair hypervisorArch : hypervisorsArchInUse) { @@ -822,11 +800,12 @@ private void validateTemplates(List> hyp templatesFound = false; break; } - if (CPU.archARM64Identifier.equals(matchedTemplate.getArch())) { - LOGGER.debug("Skipping checksum comparison for the template file and metadata as the arch for template is {}", - matchedTemplate.getArch()); + File tempFile = getTemplateFile(matchedTemplate); + if (tempFile == null) { + LOGGER.warn("Failed to download template for hypervisor {} and arch {}, moving ahead", + matchedTemplate.getHypervisorType().name(), matchedTemplate.getArch()); + continue; } - File tempFile = new File(TEMPLATES_PATH + matchedTemplate.getFilename()); String templateChecksum = DigestHelper.calculateChecksum(tempFile); if (!templateChecksum.equals(matchedTemplate.getChecksum())) { LOGGER.error("Checksum {} for file {} does not match checksum {} from metadata", @@ -842,7 +821,7 @@ private void validateTemplates(List> hyp } } - protected void registerTemplatesForZone(long zoneId) { + protected void registerTemplatesForZone(long zoneId, String filePath) { Pair storeUrlAndId = getNfsStoreInZone(zoneId); String nfsVersion = getNfsVersion(storeUrlAndId.second()); mountStore(storeUrlAndId.first(), filePath, nfsVersion); @@ -856,7 +835,7 @@ protected void registerTemplatesForZone(long zoneId) { TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId); if (templateDataStoreVO != null) { String installPath = templateDataStoreVO.getInstallPath(); - if (validateIfSeeded(storeUrlAndId.first(), installPath, nfsVersion)) { + if (validateIfSeeded(templateDataStoreVO, storeUrlAndId.first(), installPath, nfsVersion)) { continue; } } @@ -891,7 +870,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { if (filePath == null) { throw new CloudRuntimeException("Failed to create temporary file path to mount the store"); } - registerTemplatesForZone(zoneId); + registerTemplatesForZone(zoneId, filePath); unmountStore(filePath); } catch (Exception e) { unmountStore(filePath); diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index 4205e06aa693..c2753163f6bf 100644 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -71,12 +71,13 @@ function createMetadataFile() { declare -a templates getTemplateVersion $1 -templates=( "kvm:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "vmware:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova" - "xenserver:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2" - "hyperv:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip" - "lxc:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "ovm3:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" ) +templates=( "kvm-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" + "kvm-aarch64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-aarch64-kvm.qcow2.bz2" + "vmware-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova" + "xenserver-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2" + "hyperv-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip" + "lxc-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" + "ovm3-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" ) PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" mkdir -p $PARENTPATH diff --git a/utils/src/main/java/com/cloud/utils/net/NetUtils.java b/utils/src/main/java/com/cloud/utils/net/NetUtils.java index 500e2401fca5..faed1380eac4 100644 --- a/utils/src/main/java/com/cloud/utils/net/NetUtils.java +++ b/utils/src/main/java/com/cloud/utils/net/NetUtils.java @@ -20,9 +20,12 @@ package com.cloud.utils.net; import java.io.BufferedReader; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.math.BigInteger; +import java.net.HttpURLConnection; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; @@ -31,6 +34,7 @@ import java.net.SocketException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; @@ -51,8 +55,8 @@ import org.apache.commons.net.util.SubnetUtils; import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.commons.validator.routines.RegexValidator; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.utils.IteratorUtil; import com.cloud.utils.Pair; @@ -1874,4 +1878,51 @@ public static String transformCidr(final String cidr) { final long start = (ip & startNetMask); return String.format("%s/%s", long2Ip(start), size); } + + public static boolean downloadFileWithProgress(final String fileURL, final String savePath, final Logger logger) { + HttpURLConnection httpConn = null; + try { + URL url = new URL(fileURL); + httpConn = (HttpURLConnection) url.openConnection(); + int responseCode = httpConn.getResponseCode(); + if (responseCode == HttpURLConnection.HTTP_OK) { + int contentLength = httpConn.getContentLength(); + if (contentLength < 0) { + logger.warn("Content length not provided for {}, progress updates may not be accurate", + fileURL); + } + try (InputStream inputStream = httpConn.getInputStream(); + FileOutputStream outputStream = new FileOutputStream(savePath)) { + byte[] buffer = new byte[4096]; + int bytesRead; + int downloaded = 0; + int lastReportedPercent = 0; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + downloaded += bytesRead; + if (contentLength > 0) { + int percentDownloaded = (int) ((downloaded / (double) contentLength) * 100); + // Update every 5 percent or on completion + if (percentDownloaded - lastReportedPercent >= 5 || percentDownloaded == 100) { + logger.debug("Downloaded {}% from {}", downloaded, fileURL); + lastReportedPercent = percentDownloaded; + } + } + } + } + logger.info("File {} downloaded successfully using {}.", fileURL, savePath); + } else { + logger.error("No file to download {}. Server replied with code: {}", fileURL, responseCode); + return false; + } + } catch (IOException ex) { + logger.error("Failed to download {} due to: {}", fileURL, ex.getMessage(), ex); + return false; + } finally { + if (httpConn != null) { + httpConn.disconnect(); + } + } + return true; + } } From f74355f9953e49dc690debd7dd25b65993a2da6a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 20 Feb 2025 19:56:54 +0530 Subject: [PATCH 12/32] move download method to httputils Signed-off-by: Abhishek Kumar --- .../upgrade/SystemVmTemplateRegistration.java | 3 +- .../main/java/com/cloud/utils/HttpUtils.java | 51 ++++++++++++++++++ .../java/com/cloud/utils/net/NetUtils.java | 53 +------------------ 3 files changed, 54 insertions(+), 53 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 8eb0204ee0ea..18f51fc699f6 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -77,6 +77,7 @@ import com.cloud.upgrade.dao.BasicTemplateDataStoreDaoImpl; import com.cloud.user.Account; import com.cloud.utils.DateUtil; +import com.cloud.utils.HttpUtils; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.GlobalLock; @@ -779,7 +780,7 @@ protected File getTemplateFile(MetadataTemplateDetails templateDetails) { StringUtils.isNotBlank(templateDetails.getUrl())) { LOGGER.debug("Downloading the template file {} for hypervisor {} and arch {} as it is not present", templateDetails.getUrl(), templateDetails.getHypervisorType().name(), templateDetails.getArch()); - if (!NetUtils.downloadFileWithProgress(templateDetails.getUrl(), filePath, LOGGER)) { + if (!HttpUtils.downloadFileWithProgress(templateDetails.getUrl(), filePath, LOGGER)) { return null; } return new File(filePath); diff --git a/utils/src/main/java/com/cloud/utils/HttpUtils.java b/utils/src/main/java/com/cloud/utils/HttpUtils.java index 1cbc4c79c177..3e993eaf1d86 100644 --- a/utils/src/main/java/com/cloud/utils/HttpUtils.java +++ b/utils/src/main/java/com/cloud/utils/HttpUtils.java @@ -25,7 +25,12 @@ import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; + +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; import java.util.Map; public class HttpUtils { @@ -151,4 +156,50 @@ public static boolean validateSessionKey(final HttpSession session, final Map 0) { + int percentDownloaded = (int) ((downloaded / (double) contentLength) * 100); + // Update every 5 percent or on completion + if (percentDownloaded - lastReportedPercent >= 5 || percentDownloaded == 100) { + logger.debug("Downloaded {}% from {}", downloaded, fileURL); + lastReportedPercent = percentDownloaded; + } + } + } + } + logger.info("File {} downloaded successfully using {}.", fileURL, savePath); + } else { + logger.error("No file to download {}. Server replied with code: {}", fileURL, responseCode); + return false; + } + } catch (IOException ex) { + logger.error("Failed to download {} due to: {}", fileURL, ex.getMessage(), ex); + return false; + } finally { + if (httpConn != null) { + httpConn.disconnect(); + } + } + return true; + } } diff --git a/utils/src/main/java/com/cloud/utils/net/NetUtils.java b/utils/src/main/java/com/cloud/utils/net/NetUtils.java index faed1380eac4..500e2401fca5 100644 --- a/utils/src/main/java/com/cloud/utils/net/NetUtils.java +++ b/utils/src/main/java/com/cloud/utils/net/NetUtils.java @@ -20,12 +20,9 @@ package com.cloud.utils.net; import java.io.BufferedReader; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.math.BigInteger; -import java.net.HttpURLConnection; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; @@ -34,7 +31,6 @@ import java.net.SocketException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; @@ -55,8 +51,8 @@ import org.apache.commons.net.util.SubnetUtils; import org.apache.commons.validator.routines.InetAddressValidator; import org.apache.commons.validator.routines.RegexValidator; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import com.cloud.utils.IteratorUtil; import com.cloud.utils.Pair; @@ -1878,51 +1874,4 @@ public static String transformCidr(final String cidr) { final long start = (ip & startNetMask); return String.format("%s/%s", long2Ip(start), size); } - - public static boolean downloadFileWithProgress(final String fileURL, final String savePath, final Logger logger) { - HttpURLConnection httpConn = null; - try { - URL url = new URL(fileURL); - httpConn = (HttpURLConnection) url.openConnection(); - int responseCode = httpConn.getResponseCode(); - if (responseCode == HttpURLConnection.HTTP_OK) { - int contentLength = httpConn.getContentLength(); - if (contentLength < 0) { - logger.warn("Content length not provided for {}, progress updates may not be accurate", - fileURL); - } - try (InputStream inputStream = httpConn.getInputStream(); - FileOutputStream outputStream = new FileOutputStream(savePath)) { - byte[] buffer = new byte[4096]; - int bytesRead; - int downloaded = 0; - int lastReportedPercent = 0; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - downloaded += bytesRead; - if (contentLength > 0) { - int percentDownloaded = (int) ((downloaded / (double) contentLength) * 100); - // Update every 5 percent or on completion - if (percentDownloaded - lastReportedPercent >= 5 || percentDownloaded == 100) { - logger.debug("Downloaded {}% from {}", downloaded, fileURL); - lastReportedPercent = percentDownloaded; - } - } - } - } - logger.info("File {} downloaded successfully using {}.", fileURL, savePath); - } else { - logger.error("No file to download {}. Server replied with code: {}", fileURL, responseCode); - return false; - } - } catch (IOException ex) { - logger.error("Failed to download {} due to: {}", fileURL, ex.getMessage(), ex); - return false; - } finally { - if (httpConn != null) { - httpConn.disconnect(); - } - } - return true; - } } From 9073ec13a97c7c4858a739700c85325a972b3d15 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 21 Feb 2025 14:36:23 +0530 Subject: [PATCH 13/32] remove unnecssary Signed-off-by: Abhishek Kumar --- engine/schema/pom.xml | 6 ------ engine/schema/src/main/resources/config.properties | 1 - 2 files changed, 7 deletions(-) delete mode 100644 engine/schema/src/main/resources/config.properties diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index caebdc9b412d..431c2fd2b03c 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -166,12 +166,6 @@ - - - src/main/resources - true - - diff --git a/engine/schema/src/main/resources/config.properties b/engine/schema/src/main/resources/config.properties deleted file mode 100644 index 6370df704a51..000000000000 --- a/engine/schema/src/main/resources/config.properties +++ /dev/null @@ -1 +0,0 @@ -systemvm.template.location=${project.systemvm.template.location} \ No newline at end of file From 48c9d29d959145683c38934629418c8d3893272a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 21 Feb 2025 14:58:40 +0530 Subject: [PATCH 14/32] fix build Signed-off-by: Abhishek Kumar --- .../java/com/cloud/upgrade/SystemVmTemplateRegistration.java | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 18f51fc699f6..5a220e6f22b8 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -85,7 +85,6 @@ import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.Script; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.dao.VMInstanceDaoImpl; From bc5f734b5ef7cca1753d4b142089119edb14707e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 22 Feb 2025 12:35:41 +0530 Subject: [PATCH 15/32] fix Signed-off-by: Abhishek Kumar --- .../upgrade/SystemVmTemplateRegistration.java | 47 ++++++++++++------- engine/schema/templateConfig.sh | 10 ++-- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 5a220e6f22b8..f066053d482d 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -376,9 +376,10 @@ public boolean validateIfSeeded(TemplateDataStoreVO templDataStoreVO, String url private static String getHypervisorArchKey(String hypervisorType, String arch) { if (Hypervisor.HypervisorType.KVM.name().equals(hypervisorType)) { - return String.format("%s-%s", hypervisorType, StringUtils.isBlank(arch) ? CPU.archX86_64Identifier : arch); + return String.format("%s-%s", hypervisorType.toLowerCase(), + StringUtils.isBlank(arch) ? CPU.archX86_64Identifier : arch); } - return hypervisorType; + return hypervisorType.toLowerCase(); } private static MetadataTemplateDetails getMetadataTemplateDetails(Hypervisor.HypervisorType hypervisorType, String arch) { @@ -744,25 +745,39 @@ public void registerTemplate(Hypervisor.HypervisorType hypervisor, String name, * exist a template corresponding to the current code version. */ public static String parseMetadataFile() { - try { - Ini ini = new Ini(); - ini.load(new FileReader(METADATA_FILE)); - for (Pair hypervisorType : hypervisorList) { - String key = getHypervisorArchKey(hypervisorType.first().name(), hypervisorType.second()); - Ini.Section section = ini.get(key); - NewTemplateMap.put(key, new MetadataTemplateDetails(hypervisorType.first(), section.get("templatename"), - section.get("filename"), section.get("downloadurl"), section.get("checksum"), - hypervisorType.second())); - } - Ini.Section section = ini.get("default"); - return section.get("version"); - } catch (Exception e) { - String errMsg = String.format("Failed to parse systemVM template metadata file: %s", METADATA_FILE); + String errMsg = String.format("Failed to parse systemVM template metadata file: %s", METADATA_FILE); + final Ini ini = new Ini(); + try (FileReader reader = new FileReader(METADATA_FILE)) { + ini.load(reader); + } catch (IOException e) { LOGGER.error(errMsg, e); throw new CloudRuntimeException(errMsg, e); } + if (!ini.containsKey("default")) { + errMsg = String.format("%s as unable to default section", errMsg); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + for (Pair hypervisorType : hypervisorList) { + String key = getHypervisorArchKey(hypervisorType.first().name(), hypervisorType.second()); + Ini.Section section = ini.get(key); + if (section == null) { + LOGGER.error("Failed to find details for {} in template metadata file: {}", key, METADATA_FILE); + continue; + } + NewTemplateMap.put(key, new MetadataTemplateDetails( + hypervisorType.first(), + section.get("templatename"), + section.get("filename"), + section.get("downloadurl"), + section.get("checksum"), + hypervisorType.second())); + } + Ini.Section defaultSection = ini.get("default"); + return defaultSection.get("version"); } + private static void cleanupStore(Long templateId, String filePath) { String destTempFolder = filePath + PARTIAL_TEMPLATE_FOLDER + String.valueOf(templateId); try { diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index c2753163f6bf..2b769f82865c 100644 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -73,11 +73,11 @@ declare -a templates getTemplateVersion $1 templates=( "kvm-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" "kvm-aarch64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-aarch64-kvm.qcow2.bz2" - "vmware-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova" - "xenserver-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2" - "hyperv-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip" - "lxc-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "ovm3-x86_64:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" ) + "vmware:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova" + "xenserver:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2" + "hyperv:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip" + "lxc:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" + "ovm3:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" ) PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" mkdir -p $PARENTPATH From 60dc9d9b9e91c96d5af3348044a4dea10a0c5912 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 22 Feb 2025 12:40:13 +0530 Subject: [PATCH 16/32] remove unnecessary change Signed-off-by: Abhishek Kumar --- .../com/cloud/upgrade/DatabaseUpgradeChecker.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index 5edbbdfed11e..abf860439375 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -30,7 +30,6 @@ import java.util.Arrays; import java.util.Date; import java.util.List; -import java.util.Properties; import javax.inject.Inject; @@ -444,18 +443,6 @@ protected void executeViewScripts() { public void check() { GlobalLock lock = GlobalLock.getInternLock("DatabaseUpgrade"); try { - - - Properties properties = new Properties(); - try (InputStream input = this.getClass().getClassLoader().getResourceAsStream("config.properties")) { - if (input != null) { - properties.load(input); - LOGGER.info("----------------------{}", properties.getProperty("systemvm.template.location", "unknown")); - } - } catch (IOException e) { - LOGGER.error("----------Error reading property", e); - } - LOGGER.info("Grabbing lock to check for database upgrade."); if (!lock.lock(20 * 60)) { throw new CloudRuntimeException("Unable to acquire lock to check for database integrity."); From 0ac96d605633b3c1b8c39b3d49362d0e940c4e48 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Mar 2025 13:19:01 +0530 Subject: [PATCH 17/32] fixes Signed-off-by: Abhishek Kumar --- .../java/com/cloud/dc/dao/ClusterDao.java | 7 +-- .../java/com/cloud/dc/dao/ClusterDaoImpl.java | 20 ++------ .../com/cloud/storage/dao/VMTemplateDao.java | 3 +- .../cloud/storage/dao/VMTemplateDaoImpl.java | 8 +++- .../upgrade/SystemVmTemplateRegistration.java | 47 ++++++++++++------- 5 files changed, 44 insertions(+), 41 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java index ddf503cb1f1c..158805a1424f 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java @@ -18,7 +18,6 @@ import java.util.List; import java.util.Map; -import java.util.Set; import com.cloud.cpu.CPU; import com.cloud.dc.ClusterVO; @@ -31,15 +30,11 @@ public interface ClusterDao extends GenericDao { ClusterVO findBy(String name, long podId); - List listByHyTypeWithoutGuid(String hyType); - List listByZoneId(long zoneId); List getAvailableHypervisorInZone(Long zoneId); - Set getDistinctAvailableHypervisorsAcrossClusters(); - - List> getDistinctHypervisorsArchAcrossClusters(); + List> getDistinctHypervisorsArchAcrossClusters(Long zoneId); List listByDcHyType(long dcId, String hyType); diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index f5a2547c2856..34e419da3c27 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -21,10 +21,8 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import javax.inject.Inject; @@ -149,14 +147,6 @@ public ClusterVO findBy(String name, long podId) { return findOneBy(sc); } - @Override - public List listByHyTypeWithoutGuid(String hyType) { - SearchCriteria sc = HyTypeWithoutGuidSearch.create(); - sc.setParameters("hypervisorType", hyType); - - return listBy(sc); - } - @Override public List listByDcHyType(long dcId, String hyType) { SearchCriteria sc = ZoneHyTypeSearch.create(); @@ -179,14 +169,12 @@ public List getAvailableHypervisorInZone(Long zoneId) { } @Override - public Set getDistinctAvailableHypervisorsAcrossClusters() { - return new HashSet<>(getAvailableHypervisorInZone(null)); - } - - @Override - public List> getDistinctHypervisorsArchAcrossClusters() { + public List> getDistinctHypervisorsArchAcrossClusters(Long zoneId) { List> hypervisorArchList = new ArrayList<>(); String selectSql = "SELECT DISTINCT hypervisor_type, arch FROM cloud.cluster WHERE removed IS NULL"; + if (zoneId != null) { + selectSql += " AND data_center_id=" + zoneId; + } TransactionLegacy txn = TransactionLegacy.currentTxn(); try { PreparedStatement stmt = txn.prepareAutoCloseStatement(selectSql); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 3ac514530ced..849aa927b3c2 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; +import com.cloud.cpu.CPU; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; @@ -75,7 +76,7 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findRoutingTemplate(HypervisorType type, String templateName); - VMTemplateVO findLatestTemplateByTypeAndHypervisor(HypervisorType hypervisorType, Storage.TemplateType type); + VMTemplateVO findLatestTemplateByTypeAndHypervisorAndArch(HypervisorType hypervisorType, CPU.CPUArch arch, Storage.TemplateType type); public Long countTemplatesForAccount(long accountId); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 7513848536b2..f586d0393b48 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -30,6 +30,7 @@ import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; +import com.cloud.cpu.CPU; import com.cloud.dc.dao.DataCenterDao; import com.cloud.domain.dao.DomainDao; import com.cloud.host.Host; @@ -47,6 +48,7 @@ import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.StringUtils; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; @@ -111,6 +113,7 @@ public VMTemplateDaoImpl() { LatestTemplateByHypervisorTypeSearch = createSearchBuilder(); LatestTemplateByHypervisorTypeSearch.and("hypervisorType", LatestTemplateByHypervisorTypeSearch.entity().getHypervisorType(), SearchCriteria.Op.EQ); LatestTemplateByHypervisorTypeSearch.and("templateType", LatestTemplateByHypervisorTypeSearch.entity().getTemplateType(), SearchCriteria.Op.EQ); + LatestTemplateByHypervisorTypeSearch.and("arch", LatestTemplateByHypervisorTypeSearch.entity().getArch(), SearchCriteria.Op.EQ); LatestTemplateByHypervisorTypeSearch.and("removed", LatestTemplateByHypervisorTypeSearch.entity().getRemoved(), SearchCriteria.Op.NULL); } @@ -618,10 +621,13 @@ public VMTemplateVO findRoutingTemplate(HypervisorType hType, String templateNam } @Override - public VMTemplateVO findLatestTemplateByTypeAndHypervisor(HypervisorType hypervisorType, TemplateType type) { + public VMTemplateVO findLatestTemplateByTypeAndHypervisorAndArch(HypervisorType hypervisorType, CPU.CPUArch arch, TemplateType type) { SearchCriteria sc = LatestTemplateByHypervisorTypeSearch.create(); sc.setParameters("hypervisorType", hypervisorType); sc.setParameters("templateType", type); + if (arch != null) { + sc.setParameters("arch", arch); + } Filter filter = new Filter(VMTemplateVO.class, "id", false, null, 1L); List templates = listBy(sc, filter); if (templates != null && !templates.isEmpty()) { diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index f066053d482d..a7a6b924b7c3 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -35,7 +35,6 @@ import java.util.Map; import java.util.Objects; import java.util.UUID; -import java.util.stream.Collectors; import javax.inject.Inject; @@ -181,6 +180,7 @@ private static class SystemVMTemplateDetails { ImageFormat format; Integer guestOsId; Hypervisor.HypervisorType hypervisorType; + CPU.CPUArch arch; Long storeId; Long size; Long physicalSize; @@ -190,7 +190,7 @@ private static class SystemVMTemplateDetails { SystemVMTemplateDetails(String uuid, String name, Date created, String url, String checksum, ImageFormat format, Integer guestOsId, Hypervisor.HypervisorType hypervisorType, - Long storeId) { + CPU.CPUArch arch, Long storeId) { this.uuid = uuid; this.name = name; this.created = created; @@ -199,6 +199,7 @@ private static class SystemVMTemplateDetails { this.format = format; this.guestOsId = guestOsId; this.hypervisorType = hypervisorType; + this.arch = arch; this.storeId = storeId; } @@ -242,6 +243,10 @@ public Hypervisor.HypervisorType getHypervisorType() { return hypervisorType; } + public CPU.CPUArch getArch() { + return arch; + } + public Long getStoreId() { return storeId; } @@ -477,12 +482,6 @@ public static void mountStore(String storeUrl, String path, String nfsVersion) { } } - private List fetchAllHypervisors(Long zoneId) { - List hypervisorTypes = clusterDao.getAvailableHypervisorInZone(zoneId); - List hypervisorList = hypervisorTypes.stream().distinct().map(Hypervisor.HypervisorType::name).collect(Collectors.toList()); - return hypervisorList; - } - private VMTemplateVO createTemplateObjectInDB(SystemVMTemplateDetails details) { Long templateId = vmTemplateDao.getNextInSequence(Long.class, "id"); VMTemplateVO template = new VMTemplateVO(); @@ -503,6 +502,7 @@ private VMTemplateVO createTemplateObjectInDB(SystemVMTemplateDetails details) { template.setGuestOSId(details.getGuestOsId()); template.setCrossZones(true); template.setHypervisorType(details.getHypervisorType()); + template.setArch(details.getArch()); template.setState(VirtualMachineTemplate.State.Inactive); template.setDeployAsIs(false); template = vmTemplateDao.persist(template); @@ -670,7 +670,7 @@ private Long performTemplateRegistrationOperations(Hypervisor.HypervisorType hyp String templateName = UUID.randomUUID().toString(); Date created = new Date(DateUtil.currentGMTTime().getTime()); SystemVMTemplateDetails details = new SystemVMTemplateDetails(templateName, name, created, - url, checksum, format, (int) guestOsId, hypervisor, storeId); + url, checksum, format, (int) guestOsId, hypervisor, arch, storeId); if (templateId == null) { VMTemplateVO template = createTemplateObjectInDB(details); if (template == null) { @@ -840,14 +840,20 @@ protected void registerTemplatesForZone(long zoneId, String filePath) { Pair storeUrlAndId = getNfsStoreInZone(zoneId); String nfsVersion = getNfsVersion(storeUrlAndId.second()); mountStore(storeUrlAndId.first(), filePath, nfsVersion); - List hypervisorList = fetchAllHypervisors(zoneId); - for (String hypervisor : hypervisorList) { - Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.getType(hypervisor); - MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisorType, null); + List> hypervisorArchList = + clusterDao.getDistinctHypervisorsArchAcrossClusters(zoneId); + for (Pair hypervisorArch : hypervisorArchList) { + Hypervisor.HypervisorType hypervisorType = hypervisorArch.first(); + MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisorType, + hypervisorArch.second()); + if (templateDetails == null) { + continue; + } Long templateId = getRegisteredTemplateId(templateDetails.getName()); if (templateId != null) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); - TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId); + TemplateDataStoreVO templateDataStoreVO = + templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId); if (templateDataStoreVO != null) { String installPath = templateDataStoreVO.getInstallPath(); if (validateIfSeeded(templateDataStoreVO, storeUrlAndId.first(), installPath, nfsVersion)) { @@ -855,7 +861,8 @@ protected void registerTemplatesForZone(long zoneId, String filePath) { } } if (templateVO != null) { - registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId, templateVO, templateDataStoreVO, filePath); + registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId, templateVO, + templateDataStoreVO, filePath); updateRegisteredTemplateDetails(templateId, templateDetails); continue; } @@ -939,7 +946,7 @@ public void updateSystemVmTemplates(final Connection conn) { public void doInTransactionWithoutResult(final TransactionStatus status) { List> hypervisorsInUse; try { - hypervisorsInUse = clusterDao.getDistinctHypervisorsArchAcrossClusters(); + hypervisorsInUse = clusterDao.getDistinctHypervisorsArchAcrossClusters(null); } catch (final Exception e) { LOGGER.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: {}", e.getMessage()); throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting hypervisor types from clusters", e); @@ -968,7 +975,13 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { LOGGER.warn(String.format("%s %s SystemVm template not found. Cannot upgrade system Vms hypervisor is not used, so not failing upgrade", getSystemVmTemplateVersion(), templateDetails.getHypervisorType())); // Update the latest template URLs for corresponding hypervisor - VMTemplateVO templateVO = vmTemplateDao.findLatestTemplateByTypeAndHypervisor(templateDetails.getHypervisorType(), Storage.TemplateType.SYSTEM); + CPU.CPUArch arch = null; + if (StringUtils.isNotBlank(templateDetails.getArch())) { + arch = CPU.CPUArch.fromType(templateDetails.getArch()); + } + VMTemplateVO templateVO = vmTemplateDao.findLatestTemplateByTypeAndHypervisorAndArch( + templateDetails.getHypervisorType(), arch, + Storage.TemplateType.SYSTEM); if (templateVO != null) { updateTemplateUrlAndChecksum(templateVO, templateDetails); } From 33bb5635d7c790b949683dd04ea2bef72e08395d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Mar 2025 16:13:36 +0530 Subject: [PATCH 18/32] wip: try system vm deployment with different arch templates Signed-off-by: Abhishek Kumar --- .../java/com/cloud/dc/dao/ClusterDaoImpl.java | 3 +- .../main/java/com/cloud/host/dao/HostDao.java | 3 + .../java/com/cloud/host/dao/HostDaoImpl.java | 26 ++++ .../com/cloud/storage/dao/VMTemplateDao.java | 2 + .../cloud/storage/dao/VMTemplateDaoImpl.java | 43 +++++++ .../consoleproxy/ConsoleProxyManagerImpl.java | 118 ++++++++++-------- .../SecondaryStorageManagerImpl.java | 68 ++++++---- 7 files changed, 183 insertions(+), 80 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index 34e419da3c27..d2088e1fa6b3 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -20,6 +20,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -186,7 +187,7 @@ public List> getDistinctHypervisorsArchAcrossCluste } } catch (SQLException ex) { logger.error("DB exception {}", ex.getMessage(), ex); - return null; + return Collections.emptyList(); } return hypervisorArchList; } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 003bf4a34a62..ecab7ced7313 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -19,6 +19,7 @@ import java.util.Date; import java.util.List; +import com.cloud.cpu.CPU; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; @@ -194,5 +195,7 @@ List findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long List listDistinctHypervisorTypes(final Long zoneId); + List> listDistinctHypervisorArchTypes(final Long zoneId); + List listByIds(final List ids); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 94a16497e878..7efd75844b00 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -21,6 +21,8 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -42,6 +44,7 @@ import com.cloud.cluster.agentlb.HostTransferMapVO; import com.cloud.cluster.agentlb.dao.HostTransferMapDao; import com.cloud.configuration.ManagementServiceConfiguration; +import com.cloud.cpu.CPU; import com.cloud.dc.ClusterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.gpu.dao.HostGpuGroupsDao; @@ -1756,6 +1759,29 @@ public List listDistinctHypervisorTypes(final Long zoneId) { return customSearch(sc, null); } + @Override + public List> listDistinctHypervisorArchTypes(final Long zoneId) { + List> hypervisorArchList = new ArrayList<>(); + String selectSql = "SELECT DISTINCT hypervisor_type, arch FROM cloud.host WHERE removed IS NULL"; + if (zoneId != null) { + selectSql += " AND data_center_id=" + zoneId; + } + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try { + PreparedStatement stmt = txn.prepareAutoCloseStatement(selectSql); + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + HypervisorType hypervisorType = HypervisorType.valueOf(rs.getString("hypervisor_type")); + CPU.CPUArch arch = CPU.CPUArch.fromType(rs.getString("arch")); + hypervisorArchList.add(new Pair<>(hypervisorType, arch)); + } + } catch (SQLException ex) { + logger.error("DB exception {}", ex.getMessage(), ex); + return Collections.emptyList(); + } + return hypervisorArchList; + } + @Override public List listByIds(List ids) { if (CollectionUtils.isEmpty(ids)) { diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 849aa927b3c2..f49ca7f1b99c 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -74,6 +74,8 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType); + List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType); + VMTemplateVO findRoutingTemplate(HypervisorType type, String templateName); VMTemplateVO findLatestTemplateByTypeAndHypervisorAndArch(HypervisorType hypervisorType, CPU.CPUArch arch, Storage.TemplateType type); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index f586d0393b48..7717ab26592d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -17,10 +17,13 @@ package com.cloud.storage.dao; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -48,6 +51,7 @@ import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; @@ -583,6 +587,45 @@ public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hyperv .orElse(null); } + + private List listAllReadySystemVMTemplatesWithArch(Long zoneId, HypervisorType hypervisorType) { + List> availableHypervisors = _hostDao.listDistinctHypervisorArchTypes(zoneId); + if (CollectionUtils.isEmpty(availableHypervisors)) { + return Collections.emptyList(); + } + SearchCriteria sc = readySystemTemplateSearch.create(); + sc.setParameters("templateType", Storage.TemplateType.SYSTEM); + sc.setParameters("state", VirtualMachineTemplate.State.Active); + if (hypervisorType != null && !HypervisorType.Any.equals(hypervisorType)) { + sc.setParameters("hypervisorType", List.of(hypervisorType)); + } else { + sc.setParameters("hypervisorType", + availableHypervisors.stream().map(Pair::first).collect(Collectors.toList())); + } + sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", + List.of(VMTemplateStorageResourceAssoc.Status.DOWNLOADED, + VMTemplateStorageResourceAssoc.Status.BYPASSED).toArray()); + // order by descending order of id + List templates = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null)); + Map, VMTemplateVO> uniqueTemplates = new HashMap<>(); + for (VMTemplateVO template : templates) { + Pair key = new Pair<>(template.getHypervisorType(), template.getArch()); + if (availableHypervisors.contains(key) && !uniqueTemplates.containsKey(key)) { + uniqueTemplates.put(key, template); + } + } + return new ArrayList<>(uniqueTemplates.values()); + } + + @Override + public List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType) { + List templates = listAllReadySystemVMTemplatesWithArch(zoneId, hypervisorType); + if (CollectionUtils.isEmpty(templates)) { + return null; + } + return templates; + } + @Override public VMTemplateVO findRoutingTemplate(HypervisorType hType, String templateName) { SearchCriteria sc = tmpltTypeHyperSearch2.create(); diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 6b0bbde856e1..95e500897162 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -48,7 +48,6 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; @@ -82,9 +81,9 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; +import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.info.ConsoleProxyConnectionInfo; @@ -111,10 +110,8 @@ import com.cloud.resource.UnableDeleteHostException; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolStatus; -import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; @@ -281,9 +278,9 @@ public void onLoadReport(ConsoleProxyLoadReportCommand cmd) { } @Override - public void onAgentDisconnect(long agentId, com.cloud.host.Status state) { + public void onAgentDisconnect(long agentId, Status state) { - if (state == com.cloud.host.Status.Alert || state == com.cloud.host.Status.Disconnected) { + if (state == Status.Alert || state == Status.Disconnected) { HostVO host = _hostDao.findById(agentId); if (host.getType() == Type.ConsoleProxy) { String name = host.getName(); @@ -465,7 +462,7 @@ private boolean hasPreviousSession(ConsoleProxyVO proxy, VMInstanceVO vm) { public ConsoleProxyVO startProxy(long proxyVmId, boolean ignoreRestartSetting) { try { ConsoleProxyVO proxy = consoleProxyDao.findById(proxyVmId); - if (proxy.getState() == VirtualMachine.State.Running) { + if (proxy.getState() == State.Running) { return proxy; } @@ -474,7 +471,7 @@ public ConsoleProxyVO startProxy(long proxyVmId, boolean ignoreRestartSetting) { return null; } - if (proxy.getState() == VirtualMachine.State.Stopped) { + if (proxy.getState() == State.Stopped) { virtualMachineManager.advanceStart(proxy.getUuid(), null, null); proxy = consoleProxyDao.findById(proxy.getId()); return proxy; @@ -575,12 +572,12 @@ public ConsoleProxyVO startNew(long dataCenterId) throws ConcurrentOperationExce } HypervisorType availableHypervisor = resourceManager.getAvailableHypervisor(dataCenterId); - VMTemplateVO template = vmTemplateDao.findSystemVMReadyTemplate(dataCenterId, availableHypervisor); - if (template == null) { + List templates = vmTemplateDao.findSystemVMReadyTemplates(dataCenterId, availableHypervisor); + if (CollectionUtils.isEmpty(templates)) { throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenterId); } - Map context = createProxyInstance(dataCenterId, template); + Map context = createProxyInstance(dataCenterId, templates); long proxyVmId = (Long)context.get("proxyVmId"); if (proxyVmId == 0) { @@ -673,7 +670,26 @@ protected NetworkVO getDefaultNetworkForBasicZone(DataCenter dc) { return defaultNetworks.get(0); } - protected Map createProxyInstance(long dataCenterId, VMTemplateVO template) throws ConcurrentOperationException { + protected ConsoleProxyVO createOrUpdateConsoleProxy(ConsoleProxyVO proxy, long dataCenterId, long id, String name, + ServiceOffering serviceOffering, VMTemplateVO template, Account systemAccount) { + if (proxy == null) { + proxy = new ConsoleProxyVO(id, serviceOffering.getId(), name, template.getId(), + template.getHypervisorType(), template.getGuestOSId(), dataCenterId, systemAccount.getDomainId(), + systemAccount.getId(), accountManager.getSystemUser().getId(), 0, + serviceOffering.isOfferHA()); + proxy.setDynamicallyScalable(template.isDynamicallyScalable()); + proxy.setLimitCpuUse(serviceOffering.getLimitCpuUse()); + return consoleProxyDao.persist(proxy); + } + proxy.setTemplateId(template.getId()); + proxy.setHypervisorType(template.getHypervisorType()); + proxy.setGuestOSId(template.getGuestOSId()); + proxy.setDynamicallyScalable(template.isDynamicallyScalable()); + consoleProxyDao.update(proxy.getId(), proxy); + return proxy; + } + + protected Map createProxyInstance(long dataCenterId, List templates) throws ConcurrentOperationException { long id = consoleProxyDao.getNextInSequence(Long.class, "id"); String name = VirtualMachineName.getConsoleProxyName(id, instance); @@ -702,18 +718,25 @@ protected Map createProxyInstance(long dataCenterId, VMTemplateV if (serviceOffering == null) { serviceOffering = serviceOfferingDao.findDefaultSystemOffering(ServiceOffering.consoleProxyDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId)); } - ConsoleProxyVO proxy = - new ConsoleProxyVO(id, serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, - systemAcct.getDomainId(), systemAcct.getId(), accountManager.getSystemUser().getId(), 0, serviceOffering.isOfferHA()); - proxy.setDynamicallyScalable(template.isDynamicallyScalable()); - proxy.setLimitCpuUse(serviceOffering.getLimitCpuUse()); - proxy = consoleProxyDao.persist(proxy); - try { - virtualMachineManager.allocate(name, template, serviceOffering, networks, plan, null); - } catch (InsufficientCapacityException e) { - String message = String.format("Unable to allocate proxy [%s] on zone [%s] due to [%s].", proxy.toString(), dataCenterId, e.getMessage()); - logger.warn(message, e); - throw new CloudRuntimeException(message, e); + ConsoleProxyVO proxy = null; + InsufficientCapacityException lastException = null; + for (VMTemplateVO template : templates) { + proxy = createOrUpdateConsoleProxy(proxy, dataCenterId, id, name, serviceOffering, template, systemAcct); + try { + virtualMachineManager.allocate(name, template, serviceOffering, networks, plan, null); + proxy = consoleProxyDao.findById(proxy.getId()); + lastException = null; + break; + } catch (InsufficientCapacityException e) { + String message = String.format("Unable to allocate proxy [%s] on zone [%s] with %s due to [%s].", + proxy.toString(), dataCenterId, template, e.getMessage()); + logger.warn(message, e); + lastException = e; + } + } + + if (lastException != null) { + throw new CloudRuntimeException("Failed to allocate proxy [%s] on zone [%s] with available templates"); } Map context = new HashMap<>(); @@ -737,8 +760,8 @@ public void onLoadAnswer(ConsoleProxyLoadAnswer answer) { updateConsoleProxyStatus(answer.getDetails(), answer.getProxyVmId()); } - public void handleAgentDisconnect(long agentId, com.cloud.host.Status state) { - if (state == com.cloud.host.Status.Alert || state == com.cloud.host.Status.Disconnected) { + public void handleAgentDisconnect(long agentId, Status state) { + if (state == Status.Alert || state == Status.Disconnected) { HostVO host = hostDao.findById(agentId); if (host.getType() == Type.ConsoleProxy) { String name = host.getName(); @@ -789,8 +812,8 @@ private boolean allowToLaunchNew(long dcId) { return false; } List l = - consoleProxyDao.getProxyListInStates(dcId, VirtualMachine.State.Starting, VirtualMachine.State.Running, VirtualMachine.State.Stopping, - VirtualMachine.State.Stopped, VirtualMachine.State.Migrating, VirtualMachine.State.Shutdown, VirtualMachine.State.Unknown); + consoleProxyDao.getProxyListInStates(dcId, State.Starting, State.Running, State.Stopping, + State.Stopped, State.Migrating, State.Shutdown, State.Unknown); String value = configurationDao.getValue(Config.ConsoleProxyLaunchMax.key()); int launchLimit = NumbersUtil.parseInt(value, 10); @@ -876,33 +899,20 @@ public boolean isZoneReady(Map zoneHostInfoMap, DataCenter d } ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenter.getId()); if (zoneHostInfo != null && isZoneHostReady(zoneHostInfo)) { - VMTemplateVO template = vmTemplateDao.findSystemVMReadyTemplate(dataCenter.getId(), HypervisorType.Any); - if (template == null) { + List templates = vmTemplateDao.findSystemVMReadyTemplates(dataCenter.getId(), HypervisorType.Any); + if (CollectionUtils.isEmpty(templates)) { if (logger.isDebugEnabled()) { logger.debug("System vm template is not ready at data center {}, wait until it is ready to launch console proxy vm", dataCenter); } return false; } - TemplateDataStoreVO templateHostRef; - if (template.isDirectDownload()) { - templateHostRef = templateDataStoreDao.findByTemplate(template.getId(), DataStoreRole.Image); - } else { - templateHostRef = templateDataStoreDao.findByTemplateZoneDownloadStatus(template.getId(), dataCenter.getId(), Status.DOWNLOADED); - } - - if (templateHostRef != null) { - Boolean useLocalStorage = BooleanUtils.toBoolean(ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenter.getId())); - boolean hasDatacenterStoragePoolHostInfo = consoleProxyDao.hasDatacenterStoragePoolHostInfo(dataCenter.getId(), !useLocalStorage); - if (hasDatacenterStoragePoolHostInfo) { - return true; - } else { - if (logger.isDebugEnabled()) { - logger.debug("Primary storage is not ready, wait until it is ready to launch console proxy"); - } - } + boolean useLocalStorage = BooleanUtils.toBoolean(ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenter.getId())); + boolean hasDatacenterStoragePoolHostInfo = consoleProxyDao.hasDatacenterStoragePoolHostInfo(dataCenter.getId(), !useLocalStorage); + if (hasDatacenterStoragePoolHostInfo) { + return true; } else { if (logger.isDebugEnabled()) { - logger.debug("Zone [{}] is ready, but console proxy template [{}] is not ready on secondary storage.", dataCenter, template); + logger.debug("Primary storage is not ready, wait until it is ready to launch console proxy"); } } } @@ -912,9 +922,9 @@ public boolean isZoneReady(Map zoneHostInfoMap, DataCenter d private boolean isZoneHostReady(ZoneHostInfo zoneHostInfo) { int expectedFlags; if (useStorageVm) { - expectedFlags = RunningHostInfoAgregator.ZoneHostInfo.ROUTING_HOST_MASK; + expectedFlags = ZoneHostInfo.ROUTING_HOST_MASK; } else { - expectedFlags = RunningHostInfoAgregator.ZoneHostInfo.ALL_HOST_MASK; + expectedFlags = ZoneHostInfo.ALL_HOST_MASK; } return (zoneHostInfo.getFlags() & expectedFlags) == expectedFlags; @@ -1093,7 +1103,7 @@ public boolean destroyProxy(long vmId) { proxy.setPrivateIpAddress(null); consoleProxyDao.update(proxy.getId(), proxy); consoleProxyDao.remove(vmId); - HostVO host = hostDao.findByTypeNameAndZoneId(proxy.getDataCenterId(), proxy.getHostName(), Host.Type.ConsoleProxy); + HostVO host = hostDao.findByTypeNameAndZoneId(proxy.getDataCenterId(), proxy.getHostName(), Type.ConsoleProxy); if (host != null) { logger.debug("Removing host [{}] entry for proxy [{}].", host, proxy); return hostDao.remove(host.getId()); @@ -1511,7 +1521,7 @@ public boolean isPoolReadyForScan(Long dataCenterId) { return false; } - List l = consoleProxyDao.getProxyListInStates(VirtualMachine.State.Starting, VirtualMachine.State.Stopping); + List l = consoleProxyDao.getProxyListInStates(State.Starting, State.Stopping); if (l.size() > 0) { if (logger.isDebugEnabled()) { logger.debug("Zone {} has {} console proxy VM(s) in transition state", zone, l.size()); @@ -1568,7 +1578,7 @@ public HostVO createHostVOForConnectedAgent(HostVO host, StartupCommand[] cmd) { return null; } - host.setType(com.cloud.host.Host.Type.ConsoleProxy); + host.setType(Type.ConsoleProxy); return host; } @@ -1584,7 +1594,7 @@ public DeleteHostAnswer deleteHost(HostVO host, boolean isForced, boolean isForc protected HostVO findConsoleProxyHostByName(String name) { QueryBuilder sc = QueryBuilder.create(HostVO.class); - sc.and(sc.entity().getType(), Op.EQ, Host.Type.ConsoleProxy); + sc.and(sc.entity().getType(), Op.EQ, Type.ConsoleProxy); sc.and(sc.entity().getName(), Op.EQ, name); return sc.find(); } diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index d8e44203187e..2cfcadb5a5b7 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -618,6 +618,25 @@ protected NetworkVO getDefaultNetworkForBasicZone(DataCenter dc) { return defaultNetworks.get(0); } + protected SecondaryStorageVmVO createOrUpdateSecondaryStorageVm(SecondaryStorageVmVO ssvm, long dataCenterId, + long id, String name, ServiceOffering serviceOffering, VMTemplateVO template, Account systemAccount, + SecondaryStorageVm.Role role) { + if (ssvm == null) { + ssvm = new SecondaryStorageVmVO(id, serviceOffering.getId(), name, template.getId(), + template.getHypervisorType(), template.getGuestOSId(), dataCenterId, systemAccount.getDomainId(), + systemAccount.getId(), _accountMgr.getSystemUser().getId(), role, serviceOffering.isOfferHA()); + ssvm.setDynamicallyScalable(template.isDynamicallyScalable()); + ssvm.setLimitCpuUse(serviceOffering.getLimitCpuUse()); + return _secStorageVmDao.persist(ssvm); + } + ssvm.setTemplateId(template.getId()); + ssvm.setHypervisorType(template.getHypervisorType()); + ssvm.setGuestOSId(template.getGuestOSId()); + ssvm.setDynamicallyScalable(template.isDynamicallyScalable()); + _secStorageVmDao.update(ssvm.getId(), ssvm); + return ssvm; + } + protected Map createSecStorageVmInstance(long dataCenterId, SecondaryStorageVm.Role role) { DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dataCenterId); if (secStore == null) { @@ -657,28 +676,34 @@ protected Map createSecStorageVmInstance(long dataCenterId, Seco } HypervisorType availableHypervisor = _resourceMgr.getAvailableHypervisor(dataCenterId); - VMTemplateVO template = _templateDao.findSystemVMReadyTemplate(dataCenterId, availableHypervisor); - if (template == null) { - throw new CloudRuntimeException(String.format("Unable to find the system templates or it was not downloaded in %s.", dc.toString())); + List templates = _templateDao.findSystemVMReadyTemplates(dataCenterId, availableHypervisor); + if (CollectionUtils.isEmpty(templates)) { + throw new CloudRuntimeException(String.format("Unable to find the system templates or it was not downloaded in %s.", dc)); } ServiceOfferingVO serviceOffering = _serviceOffering; if (serviceOffering == null) { serviceOffering = _offeringDao.findDefaultSystemOffering(ServiceOffering.ssvmDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId)); } - SecondaryStorageVmVO secStorageVm = - new SecondaryStorageVmVO(id, serviceOffering.getId(), name, template.getId(), template.getHypervisorType(), template.getGuestOSId(), dataCenterId, - systemAcct.getDomainId(), systemAcct.getId(), _accountMgr.getSystemUser().getId(), role, serviceOffering.isOfferHA()); - secStorageVm.setDynamicallyScalable(template.isDynamicallyScalable()); - secStorageVm.setLimitCpuUse(serviceOffering.getLimitCpuUse()); - secStorageVm = _secStorageVmDao.persist(secStorageVm); - try { - _itMgr.allocate(name, template, serviceOffering, networks, plan, null); - secStorageVm = _secStorageVmDao.findById(secStorageVm.getId()); - } catch (InsufficientCapacityException e) { - String errorMessage = String.format("Unable to allocate secondary storage VM [%s] due to [%s].", name, e.getMessage()); - logger.warn(errorMessage, e); - throw new CloudRuntimeException(errorMessage, e); + SecondaryStorageVmVO secStorageVm = null; + InsufficientCapacityException lastException = null; + for (VMTemplateVO template : templates) { + secStorageVm = createOrUpdateSecondaryStorageVm(secStorageVm, id, dataCenterId, name, serviceOffering, + template, systemAcct, role); + try { + _itMgr.allocate(name, template, serviceOffering, networks, plan, null); + secStorageVm = _secStorageVmDao.findById(secStorageVm.getId()); + lastException = null; + break; + } catch (InsufficientCapacityException e) { + String errorMessage = String.format("Unable to allocate secondary storage VM [%s] due to [%s].", name, e.getMessage()); + logger.warn(errorMessage, e); + lastException = e; + } + } + + if (lastException != null) { + throw new CloudRuntimeException("Failed to allocate secondary storage VM [%s] on zone [%s] with available templates"); } Map context = new HashMap<>(); @@ -820,8 +845,8 @@ public boolean isZoneReady(Map zoneHostInfoMap, long dataCen } ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenterId); if (zoneHostInfo != null && (zoneHostInfo.getFlags() & RunningHostInfoAgregator.ZoneHostInfo.ROUTING_HOST_MASK) != 0) { - VMTemplateVO template = _templateDao.findSystemVMReadyTemplate(dataCenterId, HypervisorType.Any); - if (template == null) { + List templates = _templateDao.findSystemVMReadyTemplates(dataCenterId, HypervisorType.Any); + if (CollectionUtils.isEmpty(templates)) { if (logger.isDebugEnabled()) { logger.debug(String.format("System VM template is not ready at zone [%s], wait until it is ready to launch secondary storage VM.", dataCenterId)); } @@ -834,13 +859,6 @@ public boolean isZoneReady(Map zoneHostInfoMap, long dataCen return false; } - if (!template.isDirectDownload() && templateMgr.getImageStore(dataCenterId, template.getId()) == null) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("No secondary storage available in zone [%s], wait until it is ready to launch secondary storage VM.", dataCenterId)); - } - return false; - } - boolean useLocalStorage = BooleanUtils.toBoolean(ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId)); boolean hasStoragePoolHostInfo = _storagePoolHostDao.hasDatacenterStoragePoolHostInfo(dataCenterId, !useLocalStorage); if (hasStoragePoolHostInfo) { From 42598f73658bb7eae08528bdb65983d57713268e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Mar 2025 16:25:39 +0530 Subject: [PATCH 19/32] fix checkstyle Signed-off-by: Abhishek Kumar --- engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java | 1 - .../src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 7efd75844b00..5dbd4e549742 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -21,7 +21,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 7717ab26592d..23cc3dd7511b 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -17,7 +17,6 @@ package com.cloud.storage.dao; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -52,7 +51,6 @@ import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.Pair; -import com.cloud.utils.StringUtils; import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; From 59f2608aff28c8684859095bdef38c558c2984e8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Mar 2025 17:14:46 +0530 Subject: [PATCH 20/32] wip: try VR deployment with arch specific template Signed-off-by: Abhishek Kumar --- .../com/cloud/storage/dao/VMTemplateDao.java | 2 + .../cloud/storage/dao/VMTemplateDaoImpl.java | 30 +++++ .../lb/InternalLoadBalancerVMManagerImpl.java | 110 +++++++++++------- .../network/router/NetworkHelperImpl.java | 83 +++++++------ 4 files changed, 149 insertions(+), 76 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index f49ca7f1b99c..b60ec4692003 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -78,6 +78,8 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findRoutingTemplate(HypervisorType type, String templateName); + List findRoutingTemplates(HypervisorType type, String templateName); + VMTemplateVO findLatestTemplateByTypeAndHypervisorAndArch(HypervisorType hypervisorType, CPU.CPUArch arch, Storage.TemplateType type); public Long countTemplatesForAccount(long accountId); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 23cc3dd7511b..8eac860ac56b 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -661,6 +661,36 @@ public VMTemplateVO findRoutingTemplate(HypervisorType hType, String templateNam } } + @Override + public List findRoutingTemplates(HypervisorType hType, String templateName) { + SearchCriteria sc = tmpltTypeHyperSearch2.create(); + sc.setParameters("templateType", TemplateType.ROUTING); + sc.setParameters("hypervisorType", hType); + sc.setParameters("state", VirtualMachineTemplate.State.Active.toString()); + if (templateName != null) { + sc.setParameters("templateName", templateName); + } + List templates = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, 1L)); + if (CollectionUtils.isEmpty(templates)) { + sc = tmpltTypeHyperSearch2.create(); + sc.setParameters("templateType", TemplateType.SYSTEM); + sc.setParameters("hypervisorType", hType); + sc.setParameters("state", VirtualMachineTemplate.State.Active.toString()); + if (templateName != null) { + sc.setParameters("templateName", templateName); + } + templates = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, 1L)); + } + Map, VMTemplateVO> uniqueTemplates = new HashMap<>(); + for (VMTemplateVO template : templates) { + Pair key = new Pair<>(template.getHypervisorType(), template.getArch()); + if (!uniqueTemplates.containsKey(key)) { + uniqueTemplates.put(key, template); + } + } + return new ArrayList<>(uniqueTemplates.values()); + } + @Override public VMTemplateVO findLatestTemplateByTypeAndHypervisorAndArch(HypervisorType hypervisorType, CPU.CPUArch arch, TemplateType type) { SearchCriteria sc = LatestTemplateByHypervisorTypeSearch.create(); diff --git a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java index 9469af7eb750..ef7f6ce8a536 100644 --- a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java +++ b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -16,6 +16,12 @@ // under the License. package org.apache.cloudstack.network.lb; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.Hyperv; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.KVM; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.LXC; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.VMware; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.XenServer; + import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -31,6 +37,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO; import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao; +import org.apache.commons.collections.CollectionUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -118,12 +125,6 @@ import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; -import static com.cloud.hypervisor.Hypervisor.HypervisorType.Hyperv; -import static com.cloud.hypervisor.Hypervisor.HypervisorType.KVM; -import static com.cloud.hypervisor.Hypervisor.HypervisorType.LXC; -import static com.cloud.hypervisor.Hypervisor.HypervisorType.VMware; -import static com.cloud.hypervisor.Hypervisor.HypervisorType.XenServer; - public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements InternalLoadBalancerVMManager, InternalLoadBalancerVMService, VirtualMachineGuru { static final private String InternalLbVmNamePrefix = "b"; @@ -732,6 +733,49 @@ public List findInternalLbVms(final long guestNetworkId, final I return internalLbVms; } + protected String getRouterTemplateForHypervisor(HypervisorType hypervisorType, long dataCenterId) { + String templateName = null; + if (XenServer.equals(hypervisorType)) { + templateName = VirtualNetworkApplianceManager.RouterTemplateXen.valueIn(dataCenterId); + } else if (KVM.equals(hypervisorType)) { + templateName = VirtualNetworkApplianceManager.RouterTemplateKvm.valueIn(dataCenterId); + } else if (VMware.equals(hypervisorType)) { + templateName = VirtualNetworkApplianceManager.RouterTemplateVmware.valueIn(dataCenterId); + } else if (Hyperv.equals(hypervisorType)) { + templateName = VirtualNetworkApplianceManager.RouterTemplateHyperV.valueIn(dataCenterId); + } else if (LXC.equals(hypervisorType)) { + templateName = VirtualNetworkApplianceManager.RouterTemplateLxc.valueIn(dataCenterId); + } + return templateName; + } + + protected DomainRouterVO deployInternalLbVmWithTemplates(final long id, final DeploymentPlan plan, final long internalLbProviderId, + final Account owner, final long userId, final Long vpcId, final ServiceOffering routerOffering, + final LinkedHashMap> networks, + final List templates) throws InsufficientCapacityException { + for (final Iterator templatesIterator = templates.iterator(); templatesIterator.hasNext();) { + final VMTemplateVO template = templatesIterator.next(); + try { + DomainRouterVO internalLbVm = new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, + VirtualMachineName.getSystemVmName(id, _instance, InternalLbVmNamePrefix), + template.getId(), template.getHypervisorType(), template.getGuestOSId(), + owner.getDomainId(), owner.getId(), userId, false, + RedundantState.UNKNOWN, false, false, VirtualMachine.Type.InternalLoadBalancerVm, vpcId); + internalLbVm.setRole(Role.INTERNAL_LB_VM); + internalLbVm = _internalLbVmDao.persist(internalLbVm); + _itMgr.allocate(internalLbVm.getInstanceName(), template, routerOffering, networks, plan, null); + return _internalLbVmDao.findById(internalLbVm.getId()); + } catch (InsufficientCapacityException ex) { + if (templatesIterator.hasNext()) { + logger.debug("Failed to allocate the VR with hypervisor {} and {}, retrying with another template", template.getHypervisorType(), template); + } else { + throw ex; + } + } + } + return null; + } + protected DomainRouterVO deployInternalLbVm(final Account owner, final DeployDestination dest, final DeploymentPlan plan, final Map params, final long internalLbProviderId, final long svcOffId, final Long vpcId, final LinkedHashMap> networks, final boolean startVm) throws ConcurrentOperationException, InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, StorageUnavailableException, @@ -743,6 +787,14 @@ protected DomainRouterVO deployInternalLbVm(final Account owner, final DeployDes // Try to allocate the internal lb twice using diff hypervisors, and when failed both times, throw the exception up final List hypervisors = getHypervisors(dest, plan, null); + long userId = CallContext.current().getCallingUserId(); + if (CallContext.current().getCallingAccount().getId() != owner.getId()) { + List userVOs = _userDao.listByAccount(owner.getAccountId()); + if (!userVOs.isEmpty()) { + userId = userVOs.get(0).getId(); + } + } + int allocateRetry = 0; int startRetry = 0; DomainRouterVO internalLbVm = null; @@ -751,45 +803,20 @@ protected DomainRouterVO deployInternalLbVm(final Account owner, final DeployDes try { final long id = _internalLbVmDao.getNextInSequence(Long.class, "id"); if (logger.isDebugEnabled()) { - logger.debug("Creating the internal lb vm " + id + " in datacenter " + dest.getDataCenter() + " with hypervisor type " + hType); + logger.debug("Creating the internal lb vm {} in datacenter {} with hypervisor type {}", + id, dest.getDataCenter(), hType); } - String templateName = null; - if (hType.equals(XenServer)) { - templateName = VirtualNetworkApplianceManager.RouterTemplateXen.valueIn(dest.getDataCenter().getId()); - } else if (hType.equals(KVM)) { - templateName = VirtualNetworkApplianceManager.RouterTemplateKvm.valueIn(dest.getDataCenter().getId()); - } else if (hType.equals(VMware)) { - templateName = VirtualNetworkApplianceManager.RouterTemplateVmware.valueIn(dest.getDataCenter().getId()); - } else if (hType.equals(Hyperv)) { - templateName = VirtualNetworkApplianceManager.RouterTemplateHyperV.valueIn(dest.getDataCenter().getId()); - } else if (hType.equals(LXC)) { - templateName = VirtualNetworkApplianceManager.RouterTemplateLxc.valueIn(dest.getDataCenter().getId()); - } - final VMTemplateVO template = _templateDao.findRoutingTemplate(hType, templateName); - - if (template == null) { - logger.debug(hType + " won't support system vm, skip it"); + final String templateName = getRouterTemplateForHypervisor(hType, dest.getDataCenter().getId()); + final List templates = _templateDao.findRoutingTemplates(hType, templateName); + if (CollectionUtils.isEmpty(templates)) { + logger.debug("{} won't support system vm, skip it", hType); continue; } - - long userId = CallContext.current().getCallingUserId(); - if (CallContext.current().getCallingAccount().getId() != owner.getId()) { - List userVOs = _userDao.listByAccount(owner.getAccountId()); - if (!userVOs.isEmpty()) { - userId = userVOs.get(0).getId(); - } - } - - internalLbVm = - new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, VirtualMachineName.getSystemVmName(id, _instance, InternalLbVmNamePrefix), - template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), userId, false, RedundantState.UNKNOWN, false, false, VirtualMachine.Type.InternalLoadBalancerVm, vpcId); - internalLbVm.setRole(Role.INTERNAL_LB_VM); - internalLbVm = _internalLbVmDao.persist(internalLbVm); - _itMgr.allocate(internalLbVm.getInstanceName(), template, routerOffering, networks, plan, null); - internalLbVm = _internalLbVmDao.findById(internalLbVm.getId()); + internalLbVm = deployInternalLbVmWithTemplates(id, plan, internalLbProviderId, owner, userId, vpcId, + routerOffering, networks, templates); } catch (final InsufficientCapacityException ex) { if (allocateRetry < 2 && iter.hasNext()) { - logger.debug("Failed to allocate the Internal lb vm with hypervisor type " + hType + ", retrying one more time"); + logger.debug("Failed to allocate the Internal lb vm with hypervisor type {}, retrying one more time", hType); continue; } else { throw ex; @@ -804,8 +831,7 @@ protected DomainRouterVO deployInternalLbVm(final Account owner, final DeployDes break; } catch (final InsufficientCapacityException ex) { if (startRetry < 2 && iter.hasNext()) { - logger.debug("Failed to start the Internal lb vm " + internalLbVm + " with hypervisor type " + hType + ", " + - "destroying it and recreating one more time"); + logger.debug("Failed to start the Internal lb vm {} with hypervisor type {}, destroying it and recreating one more time", internalLbVm, hType); // destroy the internal lb vm destroyInternalLbVm(internalLbVm.getId(), _accountMgr.getSystemAccount(), User.UID_SYSTEM); continue; diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java index f33a6c2f6322..016937fc9176 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java @@ -28,8 +28,6 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; -import com.cloud.network.vpc.dao.VpcDao; -import com.cloud.utils.validation.ChecksumUtil; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -38,8 +36,8 @@ import org.apache.cloudstack.network.router.deployment.RouterDeploymentDefinition; import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.collections.CollectionUtils; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -86,6 +84,7 @@ import com.cloud.network.router.VirtualRouter.RedundantState; import com.cloud.network.router.VirtualRouter.Role; import com.cloud.network.rules.LbStickinessMethod; +import com.cloud.network.vpc.dao.VpcDao; import com.cloud.network.vpn.Site2SiteVpnManager; import com.cloud.offering.NetworkOffering; import com.cloud.resource.ResourceManager; @@ -104,6 +103,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; +import com.cloud.utils.validation.ChecksumUtil; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.Nic; import com.cloud.vm.NicProfile; @@ -496,18 +496,58 @@ protected String retrieveTemplateName(final HypervisorType hType, final long dat return templateName; } + protected DomainRouterVO deployRouterWithTemplates(final long id, + final RouterDeploymentDefinition routerDeploymentDefinition, final Account owner, final long userId, + final ServiceOfferingVO routerOffering, final boolean offerHA, final Long vpcId, + final List templates) throws InsufficientCapacityException { + for (final Iterator templatesIterator = templates.iterator(); templatesIterator.hasNext();) { + final VMTemplateVO template = templatesIterator.next(); + try { + DomainRouterVO router = new DomainRouterVO(id, routerOffering.getId(), + routerDeploymentDefinition.getVirtualProvider().getId(), + VirtualMachineName.getRouterName(id, s_vmInstanceName), template.getId(), + template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), + userId, routerDeploymentDefinition.isRedundant(), RedundantState.UNKNOWN, offerHA, false, + vpcId); + router.setDynamicallyScalable(template.isDynamicallyScalable()); + router.setRole(Role.VIRTUAL_ROUTER); + router.setLimitCpuUse(routerOffering.getLimitCpuUse()); + router = _routerDao.persist(router); + reallocateRouterNetworks(routerDeploymentDefinition, router, template, null); + return _routerDao.findById(router.getId()); + } catch (InsufficientCapacityException ex) { + if (templatesIterator.hasNext()) { + logger.debug("Failed to allocate the VR with hypervisor {} and {}, retrying with another template", template.getHypervisorType(), template); + } else { + throw ex; + } + } + } + return null; + } + @Override public DomainRouterVO deployRouter(final RouterDeploymentDefinition routerDeploymentDefinition, final boolean startRouter) throws InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, StorageUnavailableException, ResourceUnavailableException { final ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(routerDeploymentDefinition.getServiceOfferingId()); + final boolean offerHA = routerOffering.isOfferHA(); final Account owner = routerDeploymentDefinition.getOwner(); + final Long vpcId = routerDeploymentDefinition.getVpc() != null ? routerDeploymentDefinition.getVpc().getId() : null; // Router is the network element, we don't know the hypervisor type yet. // Try to allocate the domR twice using diff hypervisors, and when // failed both times, throw the exception up final List hypervisors = getHypervisors(routerDeploymentDefinition); + long userId = CallContext.current().getCallingUserId(); + if (CallContext.current().getCallingAccount().getId() != owner.getId()) { + final List userVOs = _userDao.listByAccount(owner.getAccountId()); + if (!userVOs.isEmpty()) { + userId = userVOs.get(0).getId(); + } + } + DomainRouterVO router = null; for (final Iterator iter = hypervisors.iterator(); iter.hasNext();) { final HypervisorType hType = iter.next(); @@ -521,41 +561,16 @@ public DomainRouterVO deployRouter(final RouterDeploymentDefinition routerDeploy } final String templateName = retrieveTemplateName(hType, routerDeploymentDefinition.getDest().getDataCenter().getId()); - final VMTemplateVO template = _templateDao.findRoutingTemplate(hType, templateName); - - if (template == null) { - logger.debug(hType + " won't support system vm, skip it"); + final List templates = _templateDao.findRoutingTemplates(hType, templateName); + if (CollectionUtils.isEmpty(templates)) { + logger.debug("{} won't support system vm, skip it", hType); continue; } - - final boolean offerHA = routerOffering.isOfferHA(); - - // routerDeploymentDefinition.getVpc().getId() ==> do not use - // VPC because it is not a VPC offering. - final Long vpcId = routerDeploymentDefinition.getVpc() != null ? routerDeploymentDefinition.getVpc().getId() : null; - - long userId = CallContext.current().getCallingUserId(); - if (CallContext.current().getCallingAccount().getId() != owner.getId()) { - final List userVOs = _userDao.listByAccount(owner.getAccountId()); - if (!userVOs.isEmpty()) { - userId = userVOs.get(0).getId(); - } - } - - router = new DomainRouterVO(id, routerOffering.getId(), routerDeploymentDefinition.getVirtualProvider().getId(), VirtualMachineName.getRouterName(id, - s_vmInstanceName), template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), - userId, routerDeploymentDefinition.isRedundant(), RedundantState.UNKNOWN, offerHA, false, vpcId); - - router.setDynamicallyScalable(template.isDynamicallyScalable()); - router.setRole(Role.VIRTUAL_ROUTER); - router.setLimitCpuUse(routerOffering.getLimitCpuUse()); - router = _routerDao.persist(router); - - reallocateRouterNetworks(routerDeploymentDefinition, router, template, null); - router = _routerDao.findById(router.getId()); + router = deployRouterWithTemplates(id, routerDeploymentDefinition, owner, userId, routerOffering, + offerHA, vpcId, templates); } catch (final InsufficientCapacityException ex) { if (iter.hasNext()) { - logger.debug("Failed to allocate the VR with hypervisor type " + hType + ", retrying one more time"); + logger.debug("Failed to allocate the VR with {}, retrying with another hypervisor", hType); continue; } else { throw ex; From a2a7f5dcfbe0b8e2759b91835a32e178a9474477 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 10 Mar 2025 18:14:49 +0530 Subject: [PATCH 21/32] fix systemvm deployment retry Check plan for a created instance so retry can be done if it fails for a particular template Signed-off-by: Abhishek Kumar --- .../cloud/storage/dao/VMTemplateDaoImpl.java | 10 ++++--- .../consoleproxy/ConsoleProxyManagerImpl.java | 25 +++++++++-------- .../SecondaryStorageManagerImpl.java | 27 +++++++++++-------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 8eac860ac56b..153a504b5fd2 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -18,11 +18,11 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -595,10 +595,10 @@ private List listAllReadySystemVMTemplatesWithArch(Long zoneId, Hy sc.setParameters("templateType", Storage.TemplateType.SYSTEM); sc.setParameters("state", VirtualMachineTemplate.State.Active); if (hypervisorType != null && !HypervisorType.Any.equals(hypervisorType)) { - sc.setParameters("hypervisorType", List.of(hypervisorType)); + sc.setParameters("hypervisorType", List.of(hypervisorType).toArray()); } else { sc.setParameters("hypervisorType", - availableHypervisors.stream().map(Pair::first).collect(Collectors.toList())); + availableHypervisors.stream().map(Pair::first).distinct().toArray()); } sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", List.of(VMTemplateStorageResourceAssoc.Status.DOWNLOADED, @@ -612,7 +612,9 @@ private List listAllReadySystemVMTemplatesWithArch(Long zoneId, Hy uniqueTemplates.put(key, template); } } - return new ArrayList<>(uniqueTemplates.values()); + List result = new ArrayList<>(uniqueTemplates.values()); + result.sort(Comparator.comparing(VMTemplateVO::getId).reversed()); + return result; } @Override diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 95e500897162..c7174c7fd12e 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -74,6 +74,7 @@ import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlanner; +import com.cloud.deploy.DeploymentPlanningManager; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; @@ -145,6 +146,7 @@ import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -227,6 +229,8 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy private CAManager caManager; @Inject private NetworkOrchestrationService networkMgr; + @Inject + DeploymentPlanningManager deploymentPlanningManager; private ConsoleProxyListener consoleProxyListener; @@ -719,26 +723,25 @@ protected Map createProxyInstance(long dataCenterId, List templateIterator = templates.iterator(); templateIterator.hasNext();) { + VMTemplateVO template = templateIterator.next(); proxy = createOrUpdateConsoleProxy(proxy, dataCenterId, id, name, serviceOffering, template, systemAcct); try { virtualMachineManager.allocate(name, template, serviceOffering, networks, plan, null); proxy = consoleProxyDao.findById(proxy.getId()); - lastException = null; + final VirtualMachineProfileImpl vmProfile = + new VirtualMachineProfileImpl(proxy, template, serviceOffering, systemAcct, null); + deploymentPlanningManager.planDeployment(vmProfile, plan, new DeploymentPlanner.ExcludeList(), null); break; } catch (InsufficientCapacityException e) { - String message = String.format("Unable to allocate proxy [%s] on zone [%s] with %s due to [%s].", - proxy.toString(), dataCenterId, template, e.getMessage()); - logger.warn(message, e); - lastException = e; + if (templateIterator.hasNext()) { + logger.debug("Unable to allocate proxy {} with {} in {} due to [{}]. Retrying with another template", proxy, template, dc, e.getMessage(), e); + continue; + } + throw new CloudRuntimeException("Failed to allocate proxy [%s] in zone [%s] with available templates", e); } } - if (lastException != null) { - throw new CloudRuntimeException("Failed to allocate proxy [%s] on zone [%s] with available templates"); - } - Map context = new HashMap<>(); context.put("dc", dc); HostPodVO pod = hostPodDao.findById(proxy.getPodIdToDeployIn()); diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 2cfcadb5a5b7..9608aa4648cd 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -82,6 +83,7 @@ import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlanner; +import com.cloud.deploy.DeploymentPlanningManager; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; @@ -155,6 +157,7 @@ import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.SecondaryStorageVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -254,6 +257,8 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar private IndirectAgentLB indirectAgentLB; @Inject private CAManager caManager; + @Inject + DeploymentPlanningManager deploymentPlanningManager; private int _secStorageVmMtuSize; private String _instance; @@ -686,26 +691,26 @@ protected Map createSecStorageVmInstance(long dataCenterId, Seco serviceOffering = _offeringDao.findDefaultSystemOffering(ServiceOffering.ssvmDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dataCenterId)); } SecondaryStorageVmVO secStorageVm = null; - InsufficientCapacityException lastException = null; - for (VMTemplateVO template : templates) { - secStorageVm = createOrUpdateSecondaryStorageVm(secStorageVm, id, dataCenterId, name, serviceOffering, + for (final Iterator templateIterator = templates.iterator(); templateIterator.hasNext();) { + VMTemplateVO template = templateIterator.next(); + secStorageVm = createOrUpdateSecondaryStorageVm(secStorageVm, dataCenterId, id, name, serviceOffering, template, systemAcct, role); try { _itMgr.allocate(name, template, serviceOffering, networks, plan, null); secStorageVm = _secStorageVmDao.findById(secStorageVm.getId()); - lastException = null; + final VirtualMachineProfileImpl vmProfile = + new VirtualMachineProfileImpl(secStorageVm, template, serviceOffering, systemAcct, null); + deploymentPlanningManager.planDeployment(vmProfile, plan, new DeploymentPlanner.ExcludeList(), null); break; } catch (InsufficientCapacityException e) { - String errorMessage = String.format("Unable to allocate secondary storage VM [%s] due to [%s].", name, e.getMessage()); - logger.warn(errorMessage, e); - lastException = e; + if (templateIterator.hasNext()) { + logger.debug("Unable to allocate secondary storage {} with {} due to [{}]. Retrying with another template", secStorageVm, template, e.getMessage(), e); + continue; + } + throw new CloudRuntimeException("Failed to allocate secondary storage VM [%s] in zone [%s] with available templates", e); } } - if (lastException != null) { - throw new CloudRuntimeException("Failed to allocate secondary storage VM [%s] on zone [%s] with available templates"); - } - Map context = new HashMap<>(); context.put("secStorageVmId", secStorageVm.getId()); return context; From 582187ad340b85a8755ba08f97f761ca837462d1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 11 Mar 2025 16:59:11 +0530 Subject: [PATCH 22/32] preferred arch config New zone-scope config added system.vm.preferred.architecture to allow deployment using specific architecture for system VMs including VRs. Refactor and added checks for deployements Signed-off-by: Abhishek Kumar --- .../com/cloud/vm/VirtualMachineManager.java | 4 ++ .../com/cloud/resource/ResourceManager.java | 8 +++ .../cloud/vm/VirtualMachineManagerImpl.java | 14 +++++ .../com/cloud/storage/dao/VMTemplateDao.java | 4 +- .../cloud/storage/dao/VMTemplateDaoImpl.java | 43 ++++++++++----- .../lb/InternalLoadBalancerVMManagerImpl.java | 54 ++++++++++++++---- .../consoleproxy/ConsoleProxyManagerImpl.java | 17 +++--- .../network/router/NetworkHelperImpl.java | 55 +++++++++++++------ .../cloud/resource/ResourceManagerImpl.java | 6 +- .../SecondaryStorageManagerImpl.java | 16 ++---- 10 files changed, 155 insertions(+), 66 deletions(-) diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 94c73d8f4d60..238a78e89aff 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -309,4 +309,8 @@ static String getHypervisorHostname(String name) { Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds); + void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTemplate template, + ServiceOffering serviceOffering, Account systemAccount, DeploymentPlan plan) + throws InsufficientServerCapacityException; + } diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index 8eb6462b4838..537c9b77b431 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -61,6 +61,14 @@ public interface ResourceManager extends ResourceService, Configurable { + "To force-stop VMs, choose 'ForceStop' strategy", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "Error,Migration,ForceStop"); + ConfigKey SystemVmPreferredArchitecture = new ConfigKey<>("Advanced" + , String.class + , "system.vm.preferred.architecture" + , "" + , "Preferred architecture for the system VMs including virtual routers" + , true + , ConfigKey.Scope.Zone); + /** * Register a listener for different types of resource life cycle events. * There can only be one type of listener per type of host. diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 7b231d02cb0a..c5ec1c0e850b 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -6081,4 +6081,18 @@ public Map getDiskOfferingSuitabilityForVm(long vmId, List } return result; } + + @Override + public void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTemplate template, + ServiceOffering serviceOffering, Account systemAccount, DeploymentPlan plan) + throws InsufficientServerCapacityException { + final VirtualMachineProfileImpl vmProfile = + new VirtualMachineProfileImpl(virtualMachine, template, serviceOffering, systemAccount, null); + DeployDestination destination = + _dpMgr.planDeployment(vmProfile, plan, new DeploymentPlanner.ExcludeList(), null); + if (destination == null) { + throw new InsufficientServerCapacityException(String.format("Unable to create a deployment for %s", + vmProfile), DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index b60ec4692003..1a33b2dafc2f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -74,11 +74,11 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType); - List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType); + List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType, String preferredArch); VMTemplateVO findRoutingTemplate(HypervisorType type, String templateName); - List findRoutingTemplates(HypervisorType type, String templateName); + List findRoutingTemplates(HypervisorType type, String templateName, String preferredArch); VMTemplateVO findLatestTemplateByTypeAndHypervisorAndArch(HypervisorType hypervisorType, CPU.CPUArch arch, Storage.TemplateType type); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 153a504b5fd2..6be2ca747abe 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import com.cloud.cpu.CPU; @@ -585,8 +586,31 @@ public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hyperv .orElse(null); } + private List getSortedTemplatesListWithPreferredArch( + Map, VMTemplateVO> uniqueTemplates, String preferredArch) { + List result = new ArrayList<>(uniqueTemplates.values()); + if (StringUtils.isNotBlank(preferredArch)) { + result.sort((t1, t2) -> { + boolean t1Preferred = t1.getArch().getType().equalsIgnoreCase(preferredArch); + boolean t2Preferred = t2.getArch().getType().equalsIgnoreCase(preferredArch); + if (t1Preferred && !t2Preferred) { + return -1; // t1 comes before t2 + } else if (!t1Preferred && t2Preferred) { + return 1; // t2 comes before t1 + } else { + // Both are either preferred or not preferred; use template id as a secondary sorting key. + return Long.compare(t1.getId(), t2.getId()); + } + }); + } else { + result.sort(Comparator.comparing(VMTemplateVO::getId).reversed()); + } + return result; + } - private List listAllReadySystemVMTemplatesWithArch(Long zoneId, HypervisorType hypervisorType) { + @Override + public List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType, + String preferredArch) { List> availableHypervisors = _hostDao.listDistinctHypervisorArchTypes(zoneId); if (CollectionUtils.isEmpty(availableHypervisors)) { return Collections.emptyList(); @@ -612,18 +636,7 @@ private List listAllReadySystemVMTemplatesWithArch(Long zoneId, Hy uniqueTemplates.put(key, template); } } - List result = new ArrayList<>(uniqueTemplates.values()); - result.sort(Comparator.comparing(VMTemplateVO::getId).reversed()); - return result; - } - - @Override - public List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType) { - List templates = listAllReadySystemVMTemplatesWithArch(zoneId, hypervisorType); - if (CollectionUtils.isEmpty(templates)) { - return null; - } - return templates; + return getSortedTemplatesListWithPreferredArch(uniqueTemplates, preferredArch); } @Override @@ -664,7 +677,7 @@ public VMTemplateVO findRoutingTemplate(HypervisorType hType, String templateNam } @Override - public List findRoutingTemplates(HypervisorType hType, String templateName) { + public List findRoutingTemplates(HypervisorType hType, String templateName, String preferredArch) { SearchCriteria sc = tmpltTypeHyperSearch2.create(); sc.setParameters("templateType", TemplateType.ROUTING); sc.setParameters("hypervisorType", hType); @@ -690,7 +703,7 @@ public List findRoutingTemplates(HypervisorType hType, String temp uniqueTemplates.put(key, template); } } - return new ArrayList<>(uniqueTemplates.values()); + return getSortedTemplatesListWithPreferredArch(uniqueTemplates, preferredArch); } @Override diff --git a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java index ef7f6ce8a536..0fd740e76d4e 100644 --- a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java +++ b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -749,22 +749,51 @@ protected String getRouterTemplateForHypervisor(HypervisorType hypervisorType, l return templateName; } - protected DomainRouterVO deployInternalLbVmWithTemplates(final long id, final DeploymentPlan plan, final long internalLbProviderId, - final Account owner, final long userId, final Long vpcId, final ServiceOffering routerOffering, - final LinkedHashMap> networks, - final List templates) throws InsufficientCapacityException { + protected DomainRouterVO createOrUpdateInternalLb(DomainRouterVO internalLbVm, final long id, + final long internalLbProviderId, final Account owner, final long userId, final Long vpcId, + final ServiceOffering routerOffering, + final LinkedHashMap> networks, + final VMTemplateVO template) { + if (internalLbVm == null) { + internalLbVm = new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, + VirtualMachineName.getSystemVmName(id, _instance, InternalLbVmNamePrefix), + template.getId(), template.getHypervisorType(), template.getGuestOSId(), + owner.getDomainId(), owner.getId(), userId, false, + RedundantState.UNKNOWN, false, false, + VirtualMachine.Type.InternalLoadBalancerVm, vpcId); + internalLbVm.setRole(Role.INTERNAL_LB_VM); + internalLbVm.setLimitCpuUse(routerOffering.getLimitCpuUse()); + internalLbVm.setDynamicallyScalable(template.isDynamicallyScalable()); + return _internalLbVmDao.persist(internalLbVm); + } + internalLbVm.setTemplateId(template.getId()); + internalLbVm.setDynamicallyScalable(template.isDynamicallyScalable()); + _internalLbVmDao.update(internalLbVm.getId(), internalLbVm); + return internalLbVm; + } + + protected DomainRouterVO deployInternalLbVmWithTemplates(DomainRouterVO internalLbVm, final long id, + final DeploymentPlan plan, final long internalLbProviderId, final Account owner, final long userId, + final Long vpcId, final ServiceOffering routerOffering, + final LinkedHashMap> networks, final List templates) + throws InsufficientCapacityException { for (final Iterator templatesIterator = templates.iterator(); templatesIterator.hasNext();) { final VMTemplateVO template = templatesIterator.next(); try { - DomainRouterVO internalLbVm = new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, + internalLbVm = new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, VirtualMachineName.getSystemVmName(id, _instance, InternalLbVmNamePrefix), template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), userId, false, - RedundantState.UNKNOWN, false, false, VirtualMachine.Type.InternalLoadBalancerVm, vpcId); + RedundantState.UNKNOWN, false, false, + VirtualMachine.Type.InternalLoadBalancerVm, vpcId); internalLbVm.setRole(Role.INTERNAL_LB_VM); internalLbVm = _internalLbVmDao.persist(internalLbVm); _itMgr.allocate(internalLbVm.getInstanceName(), template, routerOffering, networks, plan, null); - return _internalLbVmDao.findById(internalLbVm.getId()); + internalLbVm = _internalLbVmDao.findById(internalLbVm.getId()); + if (templatesIterator.hasNext()) { + _itMgr.checkDeploymentPlan(internalLbVm, template, routerOffering, owner, plan); + } + return internalLbVm; } catch (InsufficientCapacityException ex) { if (templatesIterator.hasNext()) { logger.debug("Failed to allocate the VR with hypervisor {} and {}, retrying with another template", template.getHypervisorType(), template); @@ -806,14 +835,17 @@ protected DomainRouterVO deployInternalLbVm(final Account owner, final DeployDes logger.debug("Creating the internal lb vm {} in datacenter {} with hypervisor type {}", id, dest.getDataCenter(), hType); } - final String templateName = getRouterTemplateForHypervisor(hType, dest.getDataCenter().getId()); - final List templates = _templateDao.findRoutingTemplates(hType, templateName); + final long zoneId = dest.getDataCenter().getId(); + final String templateName = getRouterTemplateForHypervisor(hType, zoneId); + final String preferredArch = ResourceManager.SystemVmPreferredArchitecture.valueIn(zoneId); + final List templates = _templateDao.findRoutingTemplates(hType, templateName, + preferredArch); if (CollectionUtils.isEmpty(templates)) { logger.debug("{} won't support system vm, skip it", hType); continue; } - internalLbVm = deployInternalLbVmWithTemplates(id, plan, internalLbProviderId, owner, userId, vpcId, - routerOffering, networks, templates); + internalLbVm = deployInternalLbVmWithTemplates(internalLbVm, id, plan, internalLbProviderId, owner, + userId, vpcId, routerOffering, networks, templates); } catch (final InsufficientCapacityException ex) { if (allocateRetry < 2 && iter.hasNext()) { logger.debug("Failed to allocate the Internal lb vm with hypervisor type {}, retrying one more time", hType); diff --git a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index c7174c7fd12e..5baed2643c93 100644 --- a/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -74,7 +74,6 @@ import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlanner; -import com.cloud.deploy.DeploymentPlanningManager; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; @@ -146,7 +145,6 @@ import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -229,8 +227,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy private CAManager caManager; @Inject private NetworkOrchestrationService networkMgr; - @Inject - DeploymentPlanningManager deploymentPlanningManager; private ConsoleProxyListener consoleProxyListener; @@ -576,7 +572,8 @@ public ConsoleProxyVO startNew(long dataCenterId) throws ConcurrentOperationExce } HypervisorType availableHypervisor = resourceManager.getAvailableHypervisor(dataCenterId); - List templates = vmTemplateDao.findSystemVMReadyTemplates(dataCenterId, availableHypervisor); + List templates = vmTemplateDao.findSystemVMReadyTemplates(dataCenterId, availableHypervisor, + ResourceManager.SystemVmPreferredArchitecture.valueIn(dataCenterId)); if (CollectionUtils.isEmpty(templates)) { throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenterId); } @@ -727,11 +724,10 @@ protected Map createProxyInstance(long dataCenterId, List zoneHostInfoMap, DataCenter d } ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenter.getId()); if (zoneHostInfo != null && isZoneHostReady(zoneHostInfo)) { - List templates = vmTemplateDao.findSystemVMReadyTemplates(dataCenter.getId(), HypervisorType.Any); + List templates = vmTemplateDao.findSystemVMReadyTemplates(dataCenter.getId(), + HypervisorType.Any, null); if (CollectionUtils.isEmpty(templates)) { if (logger.isDebugEnabled()) { logger.debug("System vm template is not ready at data center {}, wait until it is ready to launch console proxy vm", dataCenter); diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java index 016937fc9176..c45acf3499af 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java @@ -496,25 +496,44 @@ protected String retrieveTemplateName(final HypervisorType hType, final long dat return templateName; } - protected DomainRouterVO deployRouterWithTemplates(final long id, + protected DomainRouterVO createOrUpdateDomainRouter(DomainRouterVO router, final long id, + final RouterDeploymentDefinition routerDeploymentDefinition, final Account owner, final long userId, + final ServiceOfferingVO routerOffering, final boolean offerHA, final Long vpcId, + final VMTemplateVO template) { + if (router == null) { + router = new DomainRouterVO(id, routerOffering.getId(), + routerDeploymentDefinition.getVirtualProvider().getId(), + VirtualMachineName.getRouterName(id, s_vmInstanceName), template.getId(), + template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), + userId, routerDeploymentDefinition.isRedundant(), RedundantState.UNKNOWN, offerHA, false, + vpcId); + router.setDynamicallyScalable(template.isDynamicallyScalable()); + router.setRole(Role.VIRTUAL_ROUTER); + router.setLimitCpuUse(routerOffering.getLimitCpuUse()); + return _routerDao.persist(router); + } + router.setTemplateId(template.getId()); + router.setDynamicallyScalable(template.isDynamicallyScalable()); + _routerDao.update(router.getId(), router); + return router; + } + + protected DomainRouterVO deployRouterWithTemplates(DomainRouterVO router, final long id, final RouterDeploymentDefinition routerDeploymentDefinition, final Account owner, final long userId, final ServiceOfferingVO routerOffering, final boolean offerHA, final Long vpcId, final List templates) throws InsufficientCapacityException { for (final Iterator templatesIterator = templates.iterator(); templatesIterator.hasNext();) { final VMTemplateVO template = templatesIterator.next(); try { - DomainRouterVO router = new DomainRouterVO(id, routerOffering.getId(), - routerDeploymentDefinition.getVirtualProvider().getId(), - VirtualMachineName.getRouterName(id, s_vmInstanceName), template.getId(), - template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), - userId, routerDeploymentDefinition.isRedundant(), RedundantState.UNKNOWN, offerHA, false, - vpcId); - router.setDynamicallyScalable(template.isDynamicallyScalable()); - router.setRole(Role.VIRTUAL_ROUTER); - router.setLimitCpuUse(routerOffering.getLimitCpuUse()); - router = _routerDao.persist(router); + router = createOrUpdateDomainRouter(router, id, routerDeploymentDefinition, owner, userId, + routerOffering, offerHA, vpcId, template); reallocateRouterNetworks(routerDeploymentDefinition, router, template, null); - return _routerDao.findById(router.getId()); + router = _routerDao.findById(router.getId()); + if (templatesIterator.hasNext()) { + _itMgr.checkDeploymentPlan(router, template, routerOffering, owner, + routerDeploymentDefinition.getPlan()); + } + return router; } catch (InsufficientCapacityException ex) { if (templatesIterator.hasNext()) { logger.debug("Failed to allocate the VR with hypervisor {} and {}, retrying with another template", template.getHypervisorType(), template); @@ -559,15 +578,17 @@ public DomainRouterVO deployRouter(final RouterDeploymentDefinition routerDeploy logger.debug(String.format("Allocating the VR with id=%s in datacenter %s with the hypervisor type %s", id, routerDeploymentDefinition.getDest() .getDataCenter(), hType)); } - - final String templateName = retrieveTemplateName(hType, routerDeploymentDefinition.getDest().getDataCenter().getId()); - final List templates = _templateDao.findRoutingTemplates(hType, templateName); + final long zoneId = routerDeploymentDefinition.getDest().getDataCenter().getId(); + final String templateName = retrieveTemplateName(hType, zoneId); + final String preferredArch = ResourceManager.SystemVmPreferredArchitecture.valueIn(zoneId); + final List templates = _templateDao.findRoutingTemplates(hType, templateName, + preferredArch); if (CollectionUtils.isEmpty(templates)) { logger.debug("{} won't support system vm, skip it", hType); continue; } - router = deployRouterWithTemplates(id, routerDeploymentDefinition, owner, userId, routerOffering, - offerHA, vpcId, templates); + router = deployRouterWithTemplates(router, id, routerDeploymentDefinition, owner, userId, + routerOffering, offerHA, vpcId, templates); } catch (final InsufficientCapacityException ex) { if (iter.hasNext()) { logger.debug("Failed to allocate the VR with {}, retrying with another hypervisor", hType); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index c9ba51ce5a6f..d16e89de4778 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -3554,6 +3554,10 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {KvmSshToAgentEnabled, HOST_MAINTENANCE_LOCAL_STRATEGY}; + return new ConfigKey[] { + KvmSshToAgentEnabled, + HOST_MAINTENANCE_LOCAL_STRATEGY, + SystemVmPreferredArchitecture + }; } } diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 9608aa4648cd..04f9c476d471 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -83,7 +83,6 @@ import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlanner; -import com.cloud.deploy.DeploymentPlanningManager; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; @@ -157,7 +156,6 @@ import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.SecondaryStorageVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; @@ -257,8 +255,6 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar private IndirectAgentLB indirectAgentLB; @Inject private CAManager caManager; - @Inject - DeploymentPlanningManager deploymentPlanningManager; private int _secStorageVmMtuSize; private String _instance; @@ -681,7 +677,8 @@ protected Map createSecStorageVmInstance(long dataCenterId, Seco } HypervisorType availableHypervisor = _resourceMgr.getAvailableHypervisor(dataCenterId); - List templates = _templateDao.findSystemVMReadyTemplates(dataCenterId, availableHypervisor); + List templates = _templateDao.findSystemVMReadyTemplates(dataCenterId, availableHypervisor, + ResourceManager.SystemVmPreferredArchitecture.valueIn(dataCenterId)); if (CollectionUtils.isEmpty(templates)) { throw new CloudRuntimeException(String.format("Unable to find the system templates or it was not downloaded in %s.", dc)); } @@ -696,11 +693,9 @@ protected Map createSecStorageVmInstance(long dataCenterId, Seco secStorageVm = createOrUpdateSecondaryStorageVm(secStorageVm, dataCenterId, id, name, serviceOffering, template, systemAcct, role); try { - _itMgr.allocate(name, template, serviceOffering, networks, plan, null); + _itMgr.allocate(name, template, serviceOffering, networks, plan, template.getHypervisorType()); secStorageVm = _secStorageVmDao.findById(secStorageVm.getId()); - final VirtualMachineProfileImpl vmProfile = - new VirtualMachineProfileImpl(secStorageVm, template, serviceOffering, systemAcct, null); - deploymentPlanningManager.planDeployment(vmProfile, plan, new DeploymentPlanner.ExcludeList(), null); + _itMgr.checkDeploymentPlan(secStorageVm, template, serviceOffering, systemAcct, plan); break; } catch (InsufficientCapacityException e) { if (templateIterator.hasNext()) { @@ -850,7 +845,8 @@ public boolean isZoneReady(Map zoneHostInfoMap, long dataCen } ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenterId); if (zoneHostInfo != null && (zoneHostInfo.getFlags() & RunningHostInfoAgregator.ZoneHostInfo.ROUTING_HOST_MASK) != 0) { - List templates = _templateDao.findSystemVMReadyTemplates(dataCenterId, HypervisorType.Any); + List templates = _templateDao.findSystemVMReadyTemplates(dataCenterId, + HypervisorType.Any, null); if (CollectionUtils.isEmpty(templates)) { if (logger.isDebugEnabled()) { logger.debug(String.format("System VM template is not ready at zone [%s], wait until it is ready to launch secondary storage VM.", dataCenterId)); From 6c95e90ab53addf6b7bd90bc20d1d69344ae548f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 11 Mar 2025 13:21:36 +0530 Subject: [PATCH 23/32] ui: changes for arch auto selection in deploy vm Signed-off-by: Abhishek Kumar --- ui/src/views/compute/DeployVM.vue | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ui/src/views/compute/DeployVM.vue b/ui/src/views/compute/DeployVM.vue index 57dbe8eeabcf..df5a7b0d632e 100644 --- a/ui/src/views/compute/DeployVM.vue +++ b/ui/src/views/compute/DeployVM.vue @@ -2569,14 +2569,28 @@ export default { if (this.clusterId === null) { this.form.clusterid = undefined } - this.fetchOptions(this.params.hosts, 'hosts') + if (this.clusterId && Array.isArray(this.options.clusters)) { + const cluster = this.options.clusters.find(c => c.id === this.clusterId) + this.handleArchResourceSelected(cluster.arch) + } }, onSelectHostId (value) { this.hostId = value if (this.hostId === null) { this.form.hostid = undefined } + if (this.hostId && Array.isArray(this.options.hosts)) { + const host = this.options.hosts.find(h => h.id === this.hostId) + this.handleArchResourceSelected(host.arch) + } + }, + handleArchResourceSelected (resourceArch) { + if (!resourceArch || !this.isZoneSelectedMultiArch || this.selectedArchitecture === resourceArch) { + return + } + this.selectedArchitecture = resourceArch + this.changeArchitecture(resourceArch, this.tabKey === 'templateid') }, handleSearchFilter (name, options) { this.params[name].options = { ...this.params[name].options, ...options } From 3069929711d462d6025a0c71516a031f3cc5ae7a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 02:03:28 +0530 Subject: [PATCH 24/32] CPUArch as an enum, make config select Signed-off-by: Abhishek Kumar --- api/src/main/java/com/cloud/cpu/CPU.java | 59 +++++++++---------- .../com/cloud/resource/ResourceManager.java | 18 +++--- .../upgrade/SystemVmTemplateRegistration.java | 8 +-- .../cloudstack/util/CPUArchConverter.java | 2 +- .../com/cloud/utils/db/GenericDaoBase.java | 6 +- 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/api/src/main/java/com/cloud/cpu/CPU.java b/api/src/main/java/com/cloud/cpu/CPU.java index 4e1b9f5a5011..08cec4d86494 100644 --- a/api/src/main/java/com/cloud/cpu/CPU.java +++ b/api/src/main/java/com/cloud/cpu/CPU.java @@ -16,52 +16,49 @@ // under the License. package com.cloud.cpu; -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.commons.lang3.StringUtils; - -import java.util.LinkedHashMap; -import java.util.Map; - public class CPU { + public enum CPUArch { + x86("i686", 32), + amd64("x86_64", 64), + arm64("aarch64", 64); - public static final String archX86Identifier = "i686"; - public static final String archX86_64Identifier = "x86_64"; - public static final String archARM64Identifier = "aarch64"; - - public static class CPUArch { - private static final Map cpuArchMap = new LinkedHashMap<>(); - - public static final CPUArch archX86 = new CPUArch(archX86Identifier, 32); - public static final CPUArch amd64 = new CPUArch(archX86_64Identifier, 64); - public static final CPUArch arm64 = new CPUArch(archARM64Identifier, 64); - - private String type; - private int bits; + private final String type; + private final int bits; - public CPUArch(String type, int bits) { + CPUArch(String type, int bits) { this.type = type; this.bits = bits; - cpuArchMap.put(type, this); } public String getType() { - return this.type; + return type; } public int getBits() { - return this.bits; + return bits; + } + + public static CPUArch fromType(String identifier) { + if (identifier == null || identifier.trim().isEmpty()) { + return amd64; // Default architecture + } + for (CPUArch arch : values()) { + if (arch.type.equals(identifier)) { + return arch; + } + } + throw new IllegalArgumentException("Unsupported arch type: " + identifier); } - public static CPUArch fromType(String type) { - if (StringUtils.isBlank(type)) { - return amd64; + public static String getTypesAsCSV() { + StringBuilder sb = new StringBuilder(); + for (CPUArch arch : values()) { + sb.append(arch.getType()).append(","); } - switch (type) { - case archX86Identifier: return archX86; - case archX86_64Identifier: return amd64; - case archARM64Identifier: return arm64; - default: throw new CloudRuntimeException(String.format("Unsupported arch type: %s", type)); + if (sb.length() > 0) { + sb.setLength(sb.length() - 1); } + return sb.toString(); } } } diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index 537c9b77b431..1c81e0f945c3 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -28,6 +28,7 @@ import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.agent.api.to.GPUDeviceTO; +import com.cloud.cpu.CPU; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; import com.cloud.dc.PodCluster; @@ -61,13 +62,16 @@ public interface ResourceManager extends ResourceService, Configurable { + "To force-stop VMs, choose 'ForceStop' strategy", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "Error,Migration,ForceStop"); - ConfigKey SystemVmPreferredArchitecture = new ConfigKey<>("Advanced" - , String.class - , "system.vm.preferred.architecture" - , "" - , "Preferred architecture for the system VMs including virtual routers" - , true - , ConfigKey.Scope.Zone); + ConfigKey SystemVmPreferredArchitecture = new ConfigKey<>( + String.class, + "system.vm.preferred.architecture", + "Advanced", + "", + "Preferred architecture for the system VMs including virtual routers", + true, + ConfigKey.Scope.Global, null, null, null, null, null, + ConfigKey.Kind.Select, + "," + CPU.CPUArch.getTypesAsCSV()); /** * Register a listener for different types of resource life cycle events. diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index a7a6b924b7c3..494d59242a97 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -106,7 +106,7 @@ public class SystemVmTemplateRegistration { private static final Integer SCRIPT_TIMEOUT = 1800000; private static final Integer LOCK_WAIT_TIMEOUT = 1200; private static final List DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList( - CPU.archARM64Identifier + CPU.CPUArch.arm64.getType() ); @@ -301,8 +301,8 @@ public void setUpdated(Date updated) { } public static final List> hypervisorList = Arrays.asList( - new Pair<>(Hypervisor.HypervisorType.KVM, CPU.archX86_64Identifier), - new Pair<>(Hypervisor.HypervisorType.KVM, CPU.archARM64Identifier), + new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.x86.getType()), + new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64.getType()), new Pair<>(Hypervisor.HypervisorType.VMware, null), new Pair<>(Hypervisor.HypervisorType.XenServer, null), new Pair<>(Hypervisor.HypervisorType.Hyperv, null), @@ -382,7 +382,7 @@ public boolean validateIfSeeded(TemplateDataStoreVO templDataStoreVO, String url private static String getHypervisorArchKey(String hypervisorType, String arch) { if (Hypervisor.HypervisorType.KVM.name().equals(hypervisorType)) { return String.format("%s-%s", hypervisorType.toLowerCase(), - StringUtils.isBlank(arch) ? CPU.archX86_64Identifier : arch); + StringUtils.isBlank(arch) ? CPU.CPUArch.amd64 : arch); } return hypervisorType.toLowerCase(); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/util/CPUArchConverter.java b/engine/schema/src/main/java/org/apache/cloudstack/util/CPUArchConverter.java index e278809fb96b..8e56cce739d8 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/util/CPUArchConverter.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/util/CPUArchConverter.java @@ -21,7 +21,7 @@ import javax.persistence.AttributeConverter; import javax.persistence.Converter; -@Converter +@Converter(autoApply = true) public class CPUArchConverter implements AttributeConverter { @Override diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index bf6fb03563f8..3df119598222 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -573,6 +573,9 @@ protected void setField(Object entity, Field field, ResultSet rs, int index) thr } else { field.set(entity, rs.getLong(index)); } + } else if (field.getDeclaredAnnotation(Convert.class) != null) { + Object val = _conversionSupport.convertToEntityAttribute(field, rs.getObject(index)); + field.set(entity, val); } else if (type.isEnum()) { final Enumerated enumerated = field.getAnnotation(Enumerated.class); final EnumType enumType = (enumerated == null) ? EnumType.STRING : enumerated.value(); @@ -677,9 +680,6 @@ protected void setField(Object entity, Field field, ResultSet rs, int index) thr } } else if (type == byte[].class) { field.set(entity, rs.getBytes(index)); - } else if (field.getDeclaredAnnotation(Convert.class) != null) { - Object val = _conversionSupport.convertToEntityAttribute(field, rs.getObject(index)); - field.set(entity, val); } else { field.set(entity, rs.getObject(index)); } From c98ef69a7b54c644a0d20b17a39542de03c5dd94 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 02:00:58 +0530 Subject: [PATCH 25/32] fix persiting ilbvm Signed-off-by: Abhishek Kumar --- .../lb/InternalLoadBalancerVMManagerImpl.java | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java index 0fd740e76d4e..89ff50876822 100644 --- a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java +++ b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -751,9 +751,7 @@ protected String getRouterTemplateForHypervisor(HypervisorType hypervisorType, l protected DomainRouterVO createOrUpdateInternalLb(DomainRouterVO internalLbVm, final long id, final long internalLbProviderId, final Account owner, final long userId, final Long vpcId, - final ServiceOffering routerOffering, - final LinkedHashMap> networks, - final VMTemplateVO template) { + final ServiceOffering routerOffering, final VMTemplateVO template) { if (internalLbVm == null) { internalLbVm = new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, VirtualMachineName.getSystemVmName(id, _instance, InternalLbVmNamePrefix), @@ -780,14 +778,8 @@ protected DomainRouterVO deployInternalLbVmWithTemplates(DomainRouterVO internal for (final Iterator templatesIterator = templates.iterator(); templatesIterator.hasNext();) { final VMTemplateVO template = templatesIterator.next(); try { - internalLbVm = new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, - VirtualMachineName.getSystemVmName(id, _instance, InternalLbVmNamePrefix), - template.getId(), template.getHypervisorType(), template.getGuestOSId(), - owner.getDomainId(), owner.getId(), userId, false, - RedundantState.UNKNOWN, false, false, - VirtualMachine.Type.InternalLoadBalancerVm, vpcId); - internalLbVm.setRole(Role.INTERNAL_LB_VM); - internalLbVm = _internalLbVmDao.persist(internalLbVm); + internalLbVm = createOrUpdateInternalLb(internalLbVm, id, internalLbProviderId, owner, userId, vpcId, + routerOffering, template); _itMgr.allocate(internalLbVm.getInstanceName(), template, routerOffering, networks, plan, null); internalLbVm = _internalLbVmDao.findById(internalLbVm.getId()); if (templatesIterator.hasNext()) { From ce43205c5d821d5ec95370630c03545fb2e04fcb Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 12:03:17 +0530 Subject: [PATCH 26/32] refactor to use searchbuilder/critera for distinct pair Signed-off-by: Abhishek Kumar --- .../java/com/cloud/dc/dao/ClusterDao.java | 2 +- .../java/com/cloud/dc/dao/ClusterDaoImpl.java | 32 +++++++------------ .../java/com/cloud/host/dao/HostDaoImpl.java | 32 +++++++------------ .../upgrade/SystemVmTemplateRegistration.java | 4 +-- .../java/com/cloud/utils/db/SearchBase.java | 16 ++++++---- .../com/cloud/utils/db/SearchCriteria.java | 9 ++++-- 6 files changed, 42 insertions(+), 53 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java index 158805a1424f..fcc0a0827ee3 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java @@ -34,7 +34,7 @@ public interface ClusterDao extends GenericDao { List getAvailableHypervisorInZone(Long zoneId); - List> getDistinctHypervisorsArchAcrossClusters(Long zoneId); + List> listDistinctHypervisorsArchAcrossClusters(Long zoneId); List listByDcHyType(long dcId, String hyType); diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java index d2088e1fa6b3..f37c7911c93b 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java @@ -20,7 +20,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -170,26 +169,17 @@ public List getAvailableHypervisorInZone(Long zoneId) { } @Override - public List> getDistinctHypervisorsArchAcrossClusters(Long zoneId) { - List> hypervisorArchList = new ArrayList<>(); - String selectSql = "SELECT DISTINCT hypervisor_type, arch FROM cloud.cluster WHERE removed IS NULL"; - if (zoneId != null) { - selectSql += " AND data_center_id=" + zoneId; - } - TransactionLegacy txn = TransactionLegacy.currentTxn(); - try { - PreparedStatement stmt = txn.prepareAutoCloseStatement(selectSql); - ResultSet rs = stmt.executeQuery(); - while (rs.next()) { - HypervisorType hypervisorType = HypervisorType.valueOf(rs.getString("hypervisor_type")); - String arch = rs.getString("arch"); - hypervisorArchList.add(new Pair<>(hypervisorType, arch)); - } - } catch (SQLException ex) { - logger.error("DB exception {}", ex.getMessage(), ex); - return Collections.emptyList(); - } - return hypervisorArchList; + public List> listDistinctHypervisorsArchAcrossClusters(Long zoneId) { + SearchBuilder sb = createSearchBuilder(); + sb.select(null, Func.DISTINCT_PAIR, sb.entity().getHypervisorType(), sb.entity().getArch()); + sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("zoneId", zoneId); + final List clusters = search(sc, null); + return clusters.stream() + .map(c -> new Pair<>(c.getHypervisorType(), c.getArch().getType())) + .collect(Collectors.toList()); } @Override diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 5dbd4e549742..1aabbd66311e 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -21,7 +21,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -1760,25 +1759,18 @@ public List listDistinctHypervisorTypes(final Long zoneId) { @Override public List> listDistinctHypervisorArchTypes(final Long zoneId) { - List> hypervisorArchList = new ArrayList<>(); - String selectSql = "SELECT DISTINCT hypervisor_type, arch FROM cloud.host WHERE removed IS NULL"; - if (zoneId != null) { - selectSql += " AND data_center_id=" + zoneId; - } - TransactionLegacy txn = TransactionLegacy.currentTxn(); - try { - PreparedStatement stmt = txn.prepareAutoCloseStatement(selectSql); - ResultSet rs = stmt.executeQuery(); - while (rs.next()) { - HypervisorType hypervisorType = HypervisorType.valueOf(rs.getString("hypervisor_type")); - CPU.CPUArch arch = CPU.CPUArch.fromType(rs.getString("arch")); - hypervisorArchList.add(new Pair<>(hypervisorType, arch)); - } - } catch (SQLException ex) { - logger.error("DB exception {}", ex.getMessage(), ex); - return Collections.emptyList(); - } - return hypervisorArchList; + SearchBuilder sb = createSearchBuilder(); + sb.select(null, Func.DISTINCT_PAIR, sb.entity().getHypervisorType(), sb.entity().getArch()); + sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ); + sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("type", Type.Routing); + sc.setParameters("zoneId", zoneId); + final List hosts = search(sc, null); + return hosts.stream() + .map(h -> new Pair<>(h.getHypervisorType(), h.getArch())) + .collect(Collectors.toList()); } @Override diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 494d59242a97..89e750c23de8 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -841,7 +841,7 @@ protected void registerTemplatesForZone(long zoneId, String filePath) { String nfsVersion = getNfsVersion(storeUrlAndId.second()); mountStore(storeUrlAndId.first(), filePath, nfsVersion); List> hypervisorArchList = - clusterDao.getDistinctHypervisorsArchAcrossClusters(zoneId); + clusterDao.listDistinctHypervisorsArchAcrossClusters(zoneId); for (Pair hypervisorArch : hypervisorArchList) { Hypervisor.HypervisorType hypervisorType = hypervisorArch.first(); MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisorType, @@ -946,7 +946,7 @@ public void updateSystemVmTemplates(final Connection conn) { public void doInTransactionWithoutResult(final TransactionStatus status) { List> hypervisorsInUse; try { - hypervisorsInUse = clusterDao.getDistinctHypervisorsArchAcrossClusters(null); + hypervisorsInUse = clusterDao.listDistinctHypervisorsArchAcrossClusters(null); } catch (final Exception e) { LOGGER.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: {}", e.getMessage()); throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting hypervisor types from clusters", e); diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java b/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java index fcc9ded684d3..c4a657b12e62 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java @@ -36,6 +36,8 @@ import net.sf.cglib.proxy.Factory; import net.sf.cglib.proxy.MethodInterceptor; import net.sf.cglib.proxy.MethodProxy; + +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; /** @@ -122,7 +124,7 @@ public J select(final String fieldName, final Func func, final Object field, fin if (_entity == null) { throw new RuntimeException("SearchBuilder cannot be modified once it has been setup"); } - if (_specifiedAttrs.size() > 1) { + if (func.getCount() <= 1 && _specifiedAttrs.size() > 1) { throw new RuntimeException("You can't specify more than one field to search on"); } if (func.getCount() != -1 && (func.getCount() != (params.length + 1))) { @@ -150,7 +152,7 @@ public J select(final String fieldName, final Func func, final Object field, fin } } - final Select select = new Select(func, _specifiedAttrs.size() == 0 ? null : _specifiedAttrs.get(0), declaredField, params); + final Select select = new Select(func, _specifiedAttrs, declaredField, params); _selects.add(select); _specifiedAttrs.clear(); @@ -185,7 +187,7 @@ public J selectFields(final Object... fields) { } catch (final SecurityException e) { } catch (final NoSuchFieldException e) { } - _selects.add(new Select(Func.NATIVE, attr, field, null)); + _selects.add(new Select(Func.NATIVE, List.of(attr), field, null)); } _specifiedAttrs.clear(); @@ -528,16 +530,18 @@ public boolean equals(final Object obj) { protected static class Select { public Func func; - public Attribute attr; + public List attributes = new ArrayList<>(); public Object[] params; public Field field; protected Select() { } - public Select(final Func func, final Attribute attr, final Field field, final Object[] params) { + public Select(final Func func, final List attributes, final Field field, final Object[] params) { this.func = func; - this.attr = attr; + if (CollectionUtils.isNotEmpty(attributes)) { + this.attributes.addAll(attributes); + } this.params = params; this.field = field; } diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 8affbd5300aa..f12c2176dd3b 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; import com.cloud.utils.Pair; @@ -59,7 +60,7 @@ public int getParams() { } public enum Func { - NATIVE("@", 1), MAX("MAX(@)", 1), MIN("MIN(@)", 1), FIRST("FIRST(@)", 1), LAST("LAST(@)", 1), SUM("SUM(@)", 1), COUNT("COUNT(@)", 1), DISTINCT("DISTINCT(@)", 1); + NATIVE("@", 1), MAX("MAX(@)", 1), MIN("MIN(@)", 1), FIRST("FIRST(@)", 1), LAST("LAST(@)", 1), SUM("SUM(@)", 1), COUNT("COUNT(@)", 1), DISTINCT("DISTINCT(@)", 1), DISTINCT_PAIR("DISTINCT @, @", 2); private String func; private int count; @@ -135,10 +136,12 @@ public void getSelect(StringBuilder str, int insertAt) { for (Select select : _selects) { String func = select.func.toString() + ","; - if (select.attr == null) { + if (CollectionUtils.isEmpty(select.attributes)) { func = func.replace("@", "*"); } else { - func = func.replace("@", select.attr.table + "." + select.attr.columnName); + for (Attribute attribute : select.attributes) { + func = func.replaceFirst("@", attribute.table + "." + attribute.columnName); + } } str.insert(insertAt, func); insertAt += func.length(); From 5d696f07ebca97adfd07f27031ce2c4a3f8f51a0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 16:17:52 +0530 Subject: [PATCH 27/32] cpu test Signed-off-by: Abhishek Kumar --- api/src/main/java/com/cloud/cpu/CPU.java | 10 +-- api/src/test/java/com/cloud/cpu/CPUTest.java | 67 ++++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 api/src/test/java/com/cloud/cpu/CPUTest.java diff --git a/api/src/main/java/com/cloud/cpu/CPU.java b/api/src/main/java/com/cloud/cpu/CPU.java index 08cec4d86494..0c6036a373c3 100644 --- a/api/src/main/java/com/cloud/cpu/CPU.java +++ b/api/src/main/java/com/cloud/cpu/CPU.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.cpu; +import org.apache.commons.lang3.StringUtils; + public class CPU { public enum CPUArch { x86("i686", 32), @@ -38,16 +40,16 @@ public int getBits() { return bits; } - public static CPUArch fromType(String identifier) { - if (identifier == null || identifier.trim().isEmpty()) { + public static CPUArch fromType(String type) { + if (StringUtils.isBlank(type)) { return amd64; // Default architecture } for (CPUArch arch : values()) { - if (arch.type.equals(identifier)) { + if (arch.type.equals(type)) { return arch; } } - throw new IllegalArgumentException("Unsupported arch type: " + identifier); + throw new IllegalArgumentException("Unsupported arch type: " + type); } public static String getTypesAsCSV() { diff --git a/api/src/test/java/com/cloud/cpu/CPUTest.java b/api/src/test/java/com/cloud/cpu/CPUTest.java new file mode 100644 index 000000000000..dfedf21864cc --- /dev/null +++ b/api/src/test/java/com/cloud/cpu/CPUTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.cpu; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class CPUTest { + @Test + public void testCPUArchGetType() { + assertEquals("i686", CPU.CPUArch.x86.getType()); + assertEquals("x86_64", CPU.CPUArch.amd64.getType()); + assertEquals("aarch64", CPU.CPUArch.arm64.getType()); + } + + @Test + public void testCPUArchGetBits() { + assertEquals(32, CPU.CPUArch.x86.getBits()); + assertEquals(64, CPU.CPUArch.amd64.getBits()); + assertEquals(64, CPU.CPUArch.arm64.getBits()); + } + + @Test + public void testCPUArchFromTypeWithValidValues() { + assertEquals(CPU.CPUArch.x86, CPU.CPUArch.fromType("i686")); + assertEquals(CPU.CPUArch.amd64, CPU.CPUArch.fromType("x86_64")); + assertEquals(CPU.CPUArch.arm64, CPU.CPUArch.fromType("aarch64")); + } + + @Test + public void testCPUArchFromTypeWithDefaultForBlankOrNull() { + assertEquals(CPU.CPUArch.amd64, CPU.CPUArch.fromType("")); + assertEquals(CPU.CPUArch.amd64, CPU.CPUArch.fromType(" ")); + assertEquals(CPU.CPUArch.amd64, CPU.CPUArch.fromType(null)); + } + + @Test + public void testCPUArchFromTypeWithInvalidValue() { + Exception exception = assertThrows(IllegalArgumentException.class, () -> { + CPU.CPUArch.fromType("unsupported"); + }); + assertTrue(exception.getMessage().contains("Unsupported arch type: unsupported")); + } + + @Test + public void testCPUArchGetTypesAsCSV() { + String expectedCSV = "i686,x86_64,aarch64"; + assertEquals(expectedCSV, CPU.CPUArch.getTypesAsCSV()); + } +} From a6bdd528d6fb6e1229269fc069cfa01e545c9822 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 17:17:33 +0530 Subject: [PATCH 28/32] refactor fetch arch types Signed-off-by: Abhishek Kumar --- ui/src/components/view/SearchView.vue | 8 +------- ui/src/main.js | 4 +++- ui/src/utils/plugins.js | 12 ++++++++++++ .../image/AddKubernetesSupportedVersion.vue | 15 +-------------- ui/src/views/image/RegisterOrUploadIso.vue | 15 +-------------- ui/src/views/image/RegisterOrUploadTemplate.vue | 15 +-------------- ui/src/views/image/UpdateISO.vue | 15 +-------------- ui/src/views/image/UpdateTemplate.vue | 15 +-------------- ui/src/views/infra/ClusterAdd.vue | 17 ++--------------- 9 files changed, 23 insertions(+), 93 deletions(-) diff --git a/ui/src/components/view/SearchView.vue b/ui/src/components/view/SearchView.vue index 6738c8d5c755..57e941a5844d 100644 --- a/ui/src/components/view/SearchView.vue +++ b/ui/src/components/view/SearchView.vue @@ -451,7 +451,7 @@ export default { if (arrayField.includes('arch')) { const typeIndex = this.fields.findIndex(item => item.name === 'arch') this.fields[typeIndex].loading = true - this.fields[typeIndex].opts = this.fetchArchitectureTypes() + this.fields[typeIndex].opts = this.$fetchCpuArchitectureTypes() this.fields[typeIndex].loading = false } }, @@ -1324,12 +1324,6 @@ export default { }) }) }, - fetchArchitectureTypes () { - return [ - { id: 'x86_64', name: 'AMD 64 bits (x86_64)' }, - { id: 'aarch64', name: 'ARM 64 bits (aarch64)' } - ] - }, onSearch (value) { this.paramsFilter = {} this.searchQuery = value diff --git a/ui/src/main.js b/ui/src/main.js index 2f1d892fbd86..3fae8c6be293 100644 --- a/ui/src/main.js +++ b/ui/src/main.js @@ -35,7 +35,8 @@ import { resourceTypePlugin, fileSizeUtilPlugin, genericUtilPlugin, - localesPlugin + localesPlugin, + cpuArchitectureUtilPlugin } from './utils/plugins' import { VueAxios } from './utils/request' import directives from './utils/directives' @@ -51,6 +52,7 @@ vueApp.use(resourceTypePlugin) vueApp.use(fileSizeUtilPlugin) vueApp.use(localesPlugin) vueApp.use(genericUtilPlugin) +vueApp.use(cpuArchitectureUtilPlugin) vueApp.use(extensions) vueApp.use(directives) diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index 353fe2ac0529..7cde043aa3f5 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -523,3 +523,15 @@ export function createPathBasedOnVmType (vmtype, virtualmachineid) { return path + virtualmachineid } + +export const cpuArchitectureUtilPlugin = { + install (app) { + app.config.globalProperties.$fetchCpuArchitectureTypes = function () { + const architectures = [ + { id: 'x86_64', name: 'AMD 64 bits (x86_64)' }, + { id: 'aarch64', name: 'ARM 64 bits (aarch64)' } + ] + return architectures.map(item => ({ ...item, description: item.name })) + } + } +} \ No newline at end of file diff --git a/ui/src/views/image/AddKubernetesSupportedVersion.vue b/ui/src/views/image/AddKubernetesSupportedVersion.vue index e0cd18c8eaa5..4043ba46f7dd 100644 --- a/ui/src/views/image/AddKubernetesSupportedVersion.vue +++ b/ui/src/views/image/AddKubernetesSupportedVersion.vue @@ -216,7 +216,7 @@ export default { }) }, fetchData () { - this.fetchArchitectureTypes() + this.architectureTypes.opts = this.$fetchCpuArchitectureTypes() this.fetchZoneData() }, isValidValueForKey (obj, key) { @@ -225,19 +225,6 @@ export default { arrayHasItems (array) { return array !== null && array !== undefined && Array.isArray(array) && array.length > 0 }, - fetchArchitectureTypes () { - this.architectureTypes.opts = [] - const typesList = [] - typesList.push({ - id: 'x86_64', - description: 'AMD 64 bits (x86_64)' - }) - typesList.push({ - id: 'aarch64', - description: 'ARM 64 bits (aarch64)' - }) - this.architectureTypes.opts = typesList - }, fetchZoneData () { const params = {} params.showicon = true diff --git a/ui/src/views/image/RegisterOrUploadIso.vue b/ui/src/views/image/RegisterOrUploadIso.vue index 2c45a4dfc30a..acf4c3fd15bb 100644 --- a/ui/src/views/image/RegisterOrUploadIso.vue +++ b/ui/src/views/image/RegisterOrUploadIso.vue @@ -384,7 +384,7 @@ export default { fetchData () { this.fetchZoneData() this.fetchOsType() - this.fetchArchitectureTypes() + this.architectureTypes.opts = this.$fetchCpuArchitectureTypes() this.fetchUserData() this.fetchUserdataPolicy() if ('listDomains' in this.$store.getters.apis) { @@ -410,19 +410,6 @@ export default { this.form.zoneid = (this.zones[0].id ? this.zones[0].id : '') }) }, - fetchArchitectureTypes () { - this.architectureTypes.opts = [] - const typesList = [] - typesList.push({ - id: 'x86_64', - description: 'AMD 64 bits (x86_64)' - }) - typesList.push({ - id: 'aarch64', - description: 'ARM 64 bits (aarch64)' - }) - this.architectureTypes.opts = typesList - }, fetchOsType () { this.osTypeLoading = true diff --git a/ui/src/views/image/RegisterOrUploadTemplate.vue b/ui/src/views/image/RegisterOrUploadTemplate.vue index fb1a584b11f7..25c70a23cd75 100644 --- a/ui/src/views/image/RegisterOrUploadTemplate.vue +++ b/ui/src/views/image/RegisterOrUploadTemplate.vue @@ -583,7 +583,7 @@ export default { this.fetchZone() this.fetchOsTypes() this.fetchTemplateTypes() - this.fetchArchitectureTypes() + this.architectureTypes.opts = this.$fetchCpuArchitectureTypes() this.fetchUserData() this.fetchUserdataPolicy() if ('listDomains' in this.$store.getters.apis) { @@ -753,19 +753,6 @@ export default { } this.templateTypes.opts = templatetypes }, - fetchArchitectureTypes () { - this.architectureTypes.opts = [] - const typesList = [] - typesList.push({ - id: 'x86_64', - description: 'AMD 64 bits (x86_64)' - }) - typesList.push({ - id: 'aarch64', - description: 'ARM 64 bits (aarch64)' - }) - this.architectureTypes.opts = typesList - }, fetchUserData () { const params = {} params.listAll = true diff --git a/ui/src/views/image/UpdateISO.vue b/ui/src/views/image/UpdateISO.vue index bd0122e59fee..00e1cc536c28 100644 --- a/ui/src/views/image/UpdateISO.vue +++ b/ui/src/views/image/UpdateISO.vue @@ -216,7 +216,7 @@ export default { }, fetchData () { this.fetchOsTypes() - this.fetchArchitectureTypes() + this.architectureTypes.opts = this.$fetchCpuArchitectureTypes() this.fetchUserdata() this.fetchUserdataPolicy() }, @@ -235,19 +235,6 @@ export default { this.osTypes.loading = false }) }, - fetchArchitectureTypes () { - this.architectureTypes.opts = [] - const typesList = [] - typesList.push({ - id: 'x86_64', - description: 'AMD 64 bits (x86_64)' - }) - typesList.push({ - id: 'aarch64', - description: 'ARM 64 bits (aarch64)' - }) - this.architectureTypes.opts = typesList - }, fetchUserdataPolicy () { const userdataPolicy = [] userdataPolicy.push({ diff --git a/ui/src/views/image/UpdateTemplate.vue b/ui/src/views/image/UpdateTemplate.vue index 4c540772f89c..3a7a5a3f640a 100644 --- a/ui/src/views/image/UpdateTemplate.vue +++ b/ui/src/views/image/UpdateTemplate.vue @@ -310,7 +310,7 @@ export default { }, fetchData () { this.fetchOsTypes() - this.fetchArchitectureTypes() + this.architectureTypes.opts = this.$fetchCpuArchitectureTypes() this.fetchRootDiskControllerTypes(this.resource.hypervisor) this.fetchNicAdapterTypes() this.fetchKeyboardTypes() @@ -335,19 +335,6 @@ export default { this.osTypes.loading = false }) }, - fetchArchitectureTypes () { - this.architectureTypes.opts = [] - const typesList = [] - typesList.push({ - id: 'x86_64', - description: 'AMD 64 bits (x86_64)' - }) - typesList.push({ - id: 'aarch64', - description: 'ARM 64 bits (aarch64)' - }) - this.architectureTypes.opts = typesList - }, fetchRootDiskControllerTypes (hyperVisor) { const controller = [] this.rootDisk.opts = [] diff --git a/ui/src/views/infra/ClusterAdd.vue b/ui/src/views/infra/ClusterAdd.vue index e4f5d67a3aa6..0a55544058d5 100644 --- a/ui/src/views/infra/ClusterAdd.vue +++ b/ui/src/views/infra/ClusterAdd.vue @@ -213,7 +213,8 @@ export default { fetchData () { this.fetchZones() this.fetchHypervisors() - this.fetchArchitectureTypes() + this.architectureTypes.opts = this.$fetchCpuArchitectureTypes() + this.selectedArchitecture = this.architectureTypes?.opts?.[0]?.id || null this.params = this.$store.getters.apis.addCluster.params Object.keys(this.placeholder).forEach(item => { this.returnPlaceholder(item) }) }, @@ -240,20 +241,6 @@ export default { this.loading = false }) }, - fetchArchitectureTypes () { - this.architectureTypes.opts = [] - const typesList = [] - typesList.push({ - id: 'x86_64', - description: 'AMD 64 bits (x86_64)' - }) - typesList.push({ - id: 'aarch64', - description: 'ARM 64 bits (aarch64)' - }) - this.architectureTypes.opts = typesList - this.selectedArchitecture = this.architectureTypes.opts[0].id - }, fetchPods () { this.loading = true api('listPods', { From 87b792e55f044bef538763e2120ada6f644e3d60 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 17:22:33 +0530 Subject: [PATCH 29/32] ui,server: add s390x architecture selection Based on changes from https://github.com/apache/cloudstack/pull/10289 s390x support was added paritally with https://github.com/apache/cloudstack/pull/10038 Signed-off-by: Abhishek Kumar --- api/src/main/java/com/cloud/cpu/CPU.java | 3 ++- ui/src/utils/plugins.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/cloud/cpu/CPU.java b/api/src/main/java/com/cloud/cpu/CPU.java index 0c6036a373c3..af677649c24d 100644 --- a/api/src/main/java/com/cloud/cpu/CPU.java +++ b/api/src/main/java/com/cloud/cpu/CPU.java @@ -22,7 +22,8 @@ public class CPU { public enum CPUArch { x86("i686", 32), amd64("x86_64", 64), - arm64("aarch64", 64); + arm64("aarch64", 64), + s390x("s390x", 64); private final String type; private final int bits; diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index 7cde043aa3f5..61ad5ee34bd7 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -530,6 +530,7 @@ export const cpuArchitectureUtilPlugin = { const architectures = [ { id: 'x86_64', name: 'AMD 64 bits (x86_64)' }, { id: 'aarch64', name: 'ARM 64 bits (aarch64)' } + { id: 's390x', name: 'z/Architecture 64 bits (s390x)' } ] return architectures.map(item => ({ ...item, description: item.name })) } From 499b86142a26bf8c354c767102e11b395133e5ba Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 17:27:32 +0530 Subject: [PATCH 30/32] fix Signed-off-by: Abhishek Kumar --- ui/src/utils/plugins.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index 7cde043aa3f5..2f1c4230cb6e 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -534,4 +534,4 @@ export const cpuArchitectureUtilPlugin = { return architectures.map(item => ({ ...item, description: item.name })) } } -} \ No newline at end of file +} From a6f41efebfda5348ed985852fd897e5dc484c4fa Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 17:29:43 +0530 Subject: [PATCH 31/32] ui build fix Signed-off-by: Abhishek Kumar --- ui/src/utils/plugins.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index 9ea1c38da0ad..23bf427e2228 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -529,7 +529,7 @@ export const cpuArchitectureUtilPlugin = { app.config.globalProperties.$fetchCpuArchitectureTypes = function () { const architectures = [ { id: 'x86_64', name: 'AMD 64 bits (x86_64)' }, - { id: 'aarch64', name: 'ARM 64 bits (aarch64)' } + { id: 'aarch64', name: 'ARM 64 bits (aarch64)' }, { id: 's390x', name: 'z/Architecture 64 bits (s390x)' } ] return architectures.map(item => ({ ...item, description: item.name })) From b0cf095a849f5967523864f5ccbef91c2cc189b5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Mar 2025 18:10:52 +0530 Subject: [PATCH 32/32] fix test Signed-off-by: Abhishek Kumar --- api/src/test/java/com/cloud/cpu/CPUTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/test/java/com/cloud/cpu/CPUTest.java b/api/src/test/java/com/cloud/cpu/CPUTest.java index dfedf21864cc..4b98134e1f2c 100644 --- a/api/src/test/java/com/cloud/cpu/CPUTest.java +++ b/api/src/test/java/com/cloud/cpu/CPUTest.java @@ -61,7 +61,7 @@ public void testCPUArchFromTypeWithInvalidValue() { @Test public void testCPUArchGetTypesAsCSV() { - String expectedCSV = "i686,x86_64,aarch64"; + String expectedCSV = "i686,x86_64,aarch64,s390x"; assertEquals(expectedCSV, CPU.CPUArch.getTypesAsCSV()); } }