Skip to content

Commit 24b7c66

Browse files
DaanHooglandDaanHoogland
committed
Merge branch '4.20'
2 parents a8b18a5 + 08ad1c7 commit 24b7c66

File tree

54 files changed

+475
-236
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+475
-236
lines changed

‎api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

+7-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ public class RemoteInstanceTO implements Serializable {
2727

2828
private Hypervisor.HypervisorType hypervisorType;
2929
private String instanceName;
30+
private String instancePath;
3031

3132
// VMware Remote Instances parameters (required for exporting OVA through ovftool)
3233
// TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes
@@ -44,9 +45,10 @@ public RemoteInstanceTO(String instanceName) {
4445
this.instanceName = instanceName;
4546
}
4647

47-
public RemoteInstanceTO(String instanceName, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
48+
public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
4849
this.hypervisorType = Hypervisor.HypervisorType.VMware;
4950
this.instanceName = instanceName;
51+
this.instancePath = instancePath;
5052
this.vcenterHost = vcenterHost;
5153
this.vcenterUsername = vcenterUsername;
5254
this.vcenterPassword = vcenterPassword;
@@ -61,6 +63,10 @@ public String getInstanceName() {
6163
return this.instanceName;
6264
}
6365

66+
public String getInstancePath() {
67+
return this.instancePath;
68+
}
69+
6470
public String getVcenterUsername() {
6571
return vcenterUsername;
6672
}

‎api/src/main/java/com/cloud/storage/VolumeApiService.java

+12-10
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,12 @@
2222
import java.util.List;
2323
import java.util.Map;
2424

25+
import com.cloud.exception.ResourceAllocationException;
26+
import com.cloud.offering.DiskOffering;
27+
import com.cloud.user.Account;
2528
import com.cloud.utils.Pair;
29+
import com.cloud.utils.fsm.NoTransitionException;
30+
2631
import org.apache.cloudstack.api.command.user.volume.AssignVolumeCmd;
2732
import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
2833
import org.apache.cloudstack.api.command.user.volume.ChangeOfferingForVolumeCmd;
@@ -37,21 +42,17 @@
3742
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
3843
import org.apache.cloudstack.framework.config.ConfigKey;
3944

40-
import com.cloud.exception.ResourceAllocationException;
41-
import com.cloud.user.Account;
42-
import com.cloud.utils.fsm.NoTransitionException;
43-
4445
public interface VolumeApiService {
4546

46-
ConfigKey<Long> ConcurrentMigrationsThresholdPerDatastore = new ConfigKey<Long>("Advanced"
47+
ConfigKey<Long> ConcurrentMigrationsThresholdPerDatastore = new ConfigKey<>("Advanced"
4748
, Long.class
4849
, "concurrent.migrations.per.target.datastore"
4950
, "0"
5051
, "Limits number of migrations that can be handled per datastore concurrently; default is 0 - unlimited"
5152
, true // not sure if this is to be dynamic
5253
, ConfigKey.Scope.Global);
5354

54-
ConfigKey<Boolean> UseHttpsToUpload = new ConfigKey<Boolean>("Advanced",
55+
ConfigKey<Boolean> UseHttpsToUpload = new ConfigKey<>("Advanced",
5556
Boolean.class,
5657
"use.https.to.upload",
5758
"true",
@@ -85,7 +86,7 @@ public interface VolumeApiService {
8586
* @param cmd
8687
* the API command wrapping the criteria
8788
* @return the volume object
88-
* @throws ResourceAllocationException
89+
* @throws ResourceAllocationException no capacity to allocate the new volume size
8990
*/
9091
Volume resizeVolume(ResizeVolumeCmd cmd) throws ResourceAllocationException;
9192

@@ -139,13 +140,13 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
139140
Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName) throws ResourceAllocationException;
140141

141142
/**
142-
* Checks if the target storage supports the disk offering.
143+
* Checks if the storage pool supports the disk offering tags.
143144
* This validation is consistent with the mechanism used to select a storage pool to deploy a volume when a virtual machine is deployed or when a data disk is allocated.
144145
*
145146
* The scenarios when this method returns true or false is presented in the following table.
146147
* <table border="1">
147148
* <tr>
148-
* <th>#</th><th>Disk offering tags</th><th>Storage tags</th><th>Does the storage support the disk offering?</th>
149+
* <th>#</th><th>Disk offering diskOfferingTags</th><th>Storage diskOfferingTags</th><th>Does the storage support the disk offering?</th>
149150
* </tr>
150151
* <body>
151152
* <tr>
@@ -169,7 +170,8 @@ Volume updateVolume(long volumeId, String path, String state, Long storageId,
169170
* </body>
170171
* </table>
171172
*/
172-
boolean doesTargetStorageSupportDiskOffering(StoragePool destPool, String diskOfferingTags);
173+
boolean doesStoragePoolSupportDiskOffering(StoragePool destPool, DiskOffering diskOffering);
174+
boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags);
173175

174176
Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge);
175177

‎api/src/main/java/org/apache/cloudstack/api/command/admin/offering/IsAccountAllowedToCreateOfferingsWithTagsCmd.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
responseObject = IsAccountAllowedToCreateOfferingsWithTagsResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
3030
public class IsAccountAllowedToCreateOfferingsWithTagsCmd extends BaseCmd {
3131

32-
@Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "Account UUID")
32+
@Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "Account UUID", required = true)
3333
private Long id;
3434

3535
@Override

‎api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

+10
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ public enum PowerState {
3333

3434
private String internalCSName;
3535

36+
private String path;
37+
3638
private PowerState powerState;
3739

3840
private PowerState cloneSourcePowerState;
@@ -75,6 +77,14 @@ public void setInternalCSName(String internalCSName) {
7577
this.internalCSName = internalCSName;
7678
}
7779

80+
public String getPath() {
81+
return path;
82+
}
83+
84+
public void setPath(String path) {
85+
this.path = path;
86+
}
87+
7888
public PowerState getPowerState() {
7989
return powerState;
8090
}

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -707,7 +707,7 @@ protected void advanceExpunge(VMInstanceVO vm) throws ResourceUnavailableExcepti
707707

708708
private void handleUnsuccessfulExpungeOperation(List<Command> finalizeExpungeCommands, List<Command> nicExpungeCommands,
709709
VMInstanceVO vm, Long hostId) throws OperationTimedoutException, AgentUnavailableException {
710-
if (CollectionUtils.isNotEmpty(finalizeExpungeCommands) || CollectionUtils.isNotEmpty(nicExpungeCommands) && (hostId != null)) {
710+
if ((CollectionUtils.isNotEmpty(finalizeExpungeCommands) || CollectionUtils.isNotEmpty(nicExpungeCommands)) && hostId != null) {
711711
final Commands cmds = new Commands(Command.OnError.Stop);
712712
addAllExpungeCommandsFromList(finalizeExpungeCommands, cmds, vm);
713713
addAllExpungeCommandsFromList(nicExpungeCommands, cmds, vm);

‎engine/schema/src/main/java/com/cloud/usage/dao/UsageNetworksDaoImpl.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
// under the License.
1717
package com.cloud.usage.dao;
1818

19-
import com.cloud.network.Network;
2019
import com.cloud.usage.UsageNetworksVO;
2120
import com.cloud.utils.DateUtil;
2221
import com.cloud.utils.db.GenericDaoBase;
@@ -70,11 +69,10 @@ public void remove(long networkId, Date removed) {
7069
SearchCriteria<UsageNetworksVO> sc = this.createSearchCriteria();
7170
sc.addAnd("networkId", SearchCriteria.Op.EQ, networkId);
7271
sc.addAnd("removed", SearchCriteria.Op.NULL);
73-
UsageNetworksVO vo = findOneBy(sc);
74-
if (vo != null) {
75-
vo.setRemoved(removed);
76-
vo.setState(Network.State.Destroy.name());
77-
update(vo.getId(), vo);
72+
List<UsageNetworksVO> usageNetworksVOs = listBy(sc);
73+
for (UsageNetworksVO entry : usageNetworksVOs) {
74+
entry.setRemoved(removed);
75+
update(entry.getId(), entry);
7876
}
7977
} catch (final Exception e) {
8078
txn.rollback();

‎engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
// under the License.
1717
package com.cloud.usage.dao;
1818

19-
import com.cloud.network.vpc.Vpc;
2019
import com.cloud.usage.UsageVpcVO;
2120
import com.cloud.utils.DateUtil;
2221
import com.cloud.utils.db.GenericDaoBase;
@@ -64,11 +63,10 @@ public void remove(long vpcId, Date removed) {
6463
SearchCriteria<UsageVpcVO> sc = this.createSearchCriteria();
6564
sc.addAnd("vpcId", SearchCriteria.Op.EQ, vpcId);
6665
sc.addAnd("removed", SearchCriteria.Op.NULL);
67-
UsageVpcVO vo = findOneBy(sc);
68-
if (vo != null) {
69-
vo.setRemoved(removed);
70-
vo.setState(Vpc.State.Inactive.name());
71-
update(vo.getId(), vo);
66+
List<UsageVpcVO> usageVpcVOs = listBy(sc);
67+
for (UsageVpcVO entry : usageVpcVOs) {
68+
entry.setRemoved(removed);
69+
update(entry.getId(), entry);
7270
}
7371
} catch (final Exception e) {
7472
txn.rollback();

‎plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java

+57
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@
4545
import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap;
4646
import com.cloud.dc.dao.VmwareDatacenterDao;
4747
import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao;
48+
import com.cloud.storage.dao.VolumeDao;
49+
import com.cloud.user.User;
4850
import com.cloud.utils.Pair;
4951
import com.cloud.utils.component.AdapterBase;
5052
import com.cloud.utils.exception.CloudRuntimeException;
@@ -100,6 +102,8 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider,
100102
private AgentManager agentMgr;
101103
@Inject
102104
private VirtualMachineManager virtualMachineManager;
105+
@Inject
106+
private VolumeDao volumeDao;
103107

104108
protected VeeamClient getClient(final Long zoneId) {
105109
try {
@@ -340,6 +344,59 @@ public List<Backup.RestorePoint> listRestorePoints(VirtualMachine vm) {
340344
return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName());
341345
}
342346

347+
@Override
348+
public void syncBackups(VirtualMachine vm, Backup.Metric metric) {
349+
List<Backup.RestorePoint> restorePoints = listRestorePoints(vm);
350+
if (CollectionUtils.isEmpty(restorePoints)) {
351+
logger.debug("Can't find any restore point to VM: {}", vm);
352+
return;
353+
}
354+
Transaction.execute(new TransactionCallbackNoReturn() {
355+
@Override
356+
public void doInTransactionWithoutResult(TransactionStatus status) {
357+
final List<Backup> backupsInDb = backupDao.listByVmId(null, vm.getId());
358+
final List<Long> removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList());
359+
for (final Backup.RestorePoint restorePoint : restorePoints) {
360+
if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) {
361+
Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(backupsInDb, restorePoint, metric);
362+
if (existingBackupEntry != null) {
363+
removeList.remove(existingBackupEntry.getId());
364+
continue;
365+
}
366+
367+
BackupVO backup = new BackupVO();
368+
backup.setVmId(vm.getId());
369+
backup.setExternalId(restorePoint.getId());
370+
backup.setType(restorePoint.getType());
371+
backup.setDate(restorePoint.getCreated());
372+
backup.setStatus(Backup.Status.BackedUp);
373+
if (metric != null) {
374+
backup.setSize(metric.getBackupSize());
375+
backup.setProtectedSize(metric.getDataSize());
376+
}
377+
backup.setBackupOfferingId(vm.getBackupOfferingId());
378+
backup.setAccountId(vm.getAccountId());
379+
backup.setDomainId(vm.getDomainId());
380+
backup.setZoneId(vm.getDataCenterId());
381+
backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId())));
382+
383+
logger.debug("Creating a new entry in backups: [id: {}, uuid: {}, name: {}, vm_id: {}, external_id: {}, type: {}, date: {}, backup_offering_id: {}, account_id: {}, "
384+
+ "domain_id: {}, zone_id: {}].", backup.getId(), backup.getUuid(), backup.getName(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId());
385+
backupDao.persist(backup);
386+
387+
ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE,
388+
String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()),
389+
vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0);
390+
}
391+
}
392+
for (final Long backupIdToRemove : removeList) {
393+
logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove));
394+
backupDao.remove(backupIdToRemove);
395+
}
396+
}
397+
});
398+
}
399+
343400
@Override
344401
public String getConfigComponentName() {
345402
return BackupService.class.getSimpleName();

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serve
7070
Hypervisor.HypervisorType sourceHypervisorType = sourceInstance.getHypervisorType();
7171
String sourceInstanceName = sourceInstance.getInstanceName();
7272
Hypervisor.HypervisorType destinationHypervisorType = cmd.getDestinationHypervisorType();
73-
List<String> destinationStoragePools = cmd.getDestinationStoragePools();
7473
DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation();
7574
long timeout = (long) cmd.getWait() * 1000;
7675

@@ -193,9 +192,15 @@ private String getExportOVAUrlFromRemoteInstance(RemoteInstanceTO vmwareInstance
193192
String password = vmwareInstance.getVcenterPassword();
194193
String datacenter = vmwareInstance.getDatacenterName();
195194
String vm = vmwareInstance.getInstanceName();
195+
String path = vmwareInstance.getInstancePath();
196196

197197
String encodedUsername = encodeUsername(username);
198198
String encodedPassword = encodeUsername(password);
199+
if (StringUtils.isNotBlank(path)) {
200+
logger.info("VM path: {}", path);
201+
return String.format("vi://%s:%s@%s/%s/%s/%s",
202+
encodedUsername, encodedPassword, vcenter, datacenter, path, vm);
203+
}
199204
return String.format("vi://%s:%s@%s/%s/vm/%s",
200205
encodedUsername, encodedPassword, vcenter, datacenter, vm);
201206
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

+9-2
Original file line numberDiff line numberDiff line change
@@ -2472,8 +2472,15 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd)
24722472

24732473
template = storagePoolMgr.createPhysicalDiskFromDirectDownloadTemplate(tempFilePath, destTemplatePath, destPool, cmd.getFormat(), cmd.getWaitInMillSeconds());
24742474

2475-
String templatePath = template.getPath();
2476-
if (templatePath != null) {
2475+
String templatePath = null;
2476+
if (template != null) {
2477+
templatePath = template.getPath();
2478+
}
2479+
if (StringUtils.isEmpty(templatePath)) {
2480+
logger.warn("Skipped validation whether downloaded file is QCOW2 for template {}, due to downloaded template path is empty", template.getName());
2481+
} else if (!new File(templatePath).exists()) {
2482+
logger.warn("Skipped validation whether downloaded file is QCOW2 for template {}, due to downloaded template path is not valid: {}", template.getName(), templatePath);
2483+
} else {
24772484
try {
24782485
Qcow2Inspector.validateQcow2File(templatePath);
24792486
} catch (RuntimeException e) {

‎plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java

+24-13
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,8 @@ public class KVMHostInfo {
5858
private long reservedMemory;
5959
private long overCommitMemory;
6060
private List<String> capabilities = new ArrayList<>();
61-
62-
private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency";
6361
private static String cpuArchCommand = "/usr/bin/arch";
62+
private static List<String> cpuInfoFreqFileNames = List.of("/sys/devices/system/cpu/cpu0/cpufreq/base_frequency","/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq");
6463

6564
public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) {
6665
this.cpuSpeed = manualSpeed;
@@ -138,32 +137,44 @@ protected static long getCpuSpeed(final String cpabilities, final NodeInfo nodeI
138137
}
139138

140139
private static long getCpuSpeedFromCommandLscpu() {
140+
long speed = 0L;
141+
LOGGER.info("Fetching CPU speed from command \"lscpu\".");
141142
try {
142-
LOGGER.info("Fetching CPU speed from command \"lscpu\".");
143143
String command = "lscpu | grep -i 'Model name' | head -n 1 | egrep -o '[[:digit:]].[[:digit:]]+GHz' | sed 's/GHz//g'";
144144
if(isHostS390x()) {
145145
command = "lscpu | grep 'CPU dynamic MHz' | cut -d ':' -f 2 | tr -d ' ' | awk '{printf \"%.1f\\n\", $1 / 1000}'";
146146
}
147147
String result = Script.runSimpleBashScript(command);
148-
long speed = (long) (Float.parseFloat(result) * 1000);
148+
speed = (long) (Float.parseFloat(result) * 1000);
149149
LOGGER.info(String.format("Command [%s] resulted in the value [%s] for CPU speed.", command, speed));
150150
return speed;
151151
} catch (NullPointerException | NumberFormatException e) {
152152
LOGGER.error(String.format("Unable to retrieve the CPU speed from lscpu."), e);
153-
return 0L;
154153
}
154+
try {
155+
String command = "lscpu | grep -i 'CPU max MHz' | head -n 1 | sed 's/^.*: //' | xargs";
156+
String result = Script.runSimpleBashScript(command);
157+
speed = (long) (Float.parseFloat(result));
158+
LOGGER.info(String.format("Command [%s] resulted in the value [%s] for CPU speed.", command, speed));
159+
return speed;
160+
} catch (NullPointerException | NumberFormatException e) {
161+
LOGGER.error(String.format("Unable to retrieve the CPU speed from lscpu."), e);
162+
}
163+
return speed;
155164
}
156165

157166
private static long getCpuSpeedFromFile() {
158-
LOGGER.info(String.format("Fetching CPU speed from file [%s].", cpuInfoFreqFileName));
159-
try (Reader reader = new FileReader(cpuInfoFreqFileName)) {
160-
Long cpuInfoFreq = Long.parseLong(IOUtils.toString(reader).trim());
161-
LOGGER.info(String.format("Retrieved value [%s] from file [%s]. This corresponds to a CPU speed of [%s] MHz.", cpuInfoFreq, cpuInfoFreqFileName, cpuInfoFreq / 1000));
162-
return cpuInfoFreq / 1000;
163-
} catch (IOException | NumberFormatException e) {
164-
LOGGER.error(String.format("Unable to retrieve the CPU speed from file [%s]", cpuInfoFreqFileName), e);
165-
return 0L;
167+
for (final String cpuInfoFreqFileName: cpuInfoFreqFileNames) {
168+
LOGGER.info(String.format("Fetching CPU speed from file [%s].", cpuInfoFreqFileName));
169+
try (Reader reader = new FileReader(cpuInfoFreqFileName)) {
170+
Long cpuInfoFreq = Long.parseLong(IOUtils.toString(reader).trim());
171+
LOGGER.info(String.format("Retrieved value [%s] from file [%s]. This corresponds to a CPU speed of [%s] MHz.", cpuInfoFreq, cpuInfoFreqFileName, cpuInfoFreq / 1000));
172+
return cpuInfoFreq / 1000;
173+
} catch (IOException | NumberFormatException e) {
174+
LOGGER.error(String.format("Unable to retrieve the CPU speed from file [%s]", cpuInfoFreqFileName), e);
175+
}
166176
}
177+
return 0L;
167178
}
168179

169180
protected static long getCpuSpeedFromHostCapabilities(final String capabilities) {

0 commit comments

Comments
 (0)