Skip to content

Commit 81e052c

Browse files
DaanHooglandDaanHoogland
committed
Merge release branch 4.20 to main
* 4.20: linstor: Fix ZFS snapshot backup (#10219) fix listing of VMs by network (#10204) Configure org.eclipse.jetty.server.Request.maxFormKeys from server.properties and increase the default value (#10214) api: fix access for listSystemVmUsageHistory (#10032) Fix NPE issues during host rolling maintenance, due to host tags and custom constrained/unconstrained service offering (#9844)
2 parents 1c626c8 + 5167c3b commit 81e052c

File tree

8 files changed

+144
-11
lines changed

8 files changed

+144
-11
lines changed

‎client/conf/server.properties.in

+3
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ session.timeout=30
3232
# Max allowed API request payload/content size in bytes
3333
request.content.size=1048576
3434

35+
# Max allowed API request form keys
36+
request.max.form.keys=5000
37+
3538
# Options to configure and enable HTTPS on the management server
3639
#
3740
# For the management server to pick up these configuration settings, the configured

‎client/src/main/java/org/apache/cloudstack/ServerDaemon.java

+10
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ public class ServerDaemon implements Daemon {
8282
private static final String ACCESS_LOG = "access.log";
8383
private static final String REQUEST_CONTENT_SIZE_KEY = "request.content.size";
8484
private static final int DEFAULT_REQUEST_CONTENT_SIZE = 1048576;
85+
private static final String REQUEST_MAX_FORM_KEYS_KEY = "request.max.form.keys";
86+
private static final int DEFAULT_REQUEST_MAX_FORM_KEYS = 5000;
8587

8688
////////////////////////////////////////////////////////
8789
/////////////// Server Configuration ///////////////////
@@ -94,6 +96,7 @@ public class ServerDaemon implements Daemon {
9496
private int httpsPort = 8443;
9597
private int sessionTimeout = 30;
9698
private int maxFormContentSize = DEFAULT_REQUEST_CONTENT_SIZE;
99+
private int maxFormKeys = DEFAULT_REQUEST_MAX_FORM_KEYS;
97100
private boolean httpsEnable = false;
98101
private String accessLogFile = "access.log";
99102
private String bindInterface = null;
@@ -141,6 +144,7 @@ public void init(final DaemonContext context) {
141144
setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log"));
142145
setSessionTimeout(Integer.valueOf(properties.getProperty(SESSION_TIMEOUT, "30")));
143146
setMaxFormContentSize(Integer.valueOf(properties.getProperty(REQUEST_CONTENT_SIZE_KEY, String.valueOf(DEFAULT_REQUEST_CONTENT_SIZE))));
147+
setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS))));
144148
} catch (final IOException e) {
145149
logger.warn("Failed to read configuration from server.properties file", e);
146150
} finally {
@@ -192,6 +196,7 @@ public void start() throws Exception {
192196
// Extra config options
193197
server.setStopAtShutdown(true);
194198
server.setAttribute(ContextHandler.MAX_FORM_CONTENT_SIZE_KEY, maxFormContentSize);
199+
server.setAttribute(ContextHandler.MAX_FORM_KEYS_KEY, maxFormKeys);
195200

196201
// HTTPS Connector
197202
createHttpsConnector(httpConfig);
@@ -264,6 +269,7 @@ private Pair<SessionHandler,HandlerCollection> createHandlers() {
264269
webApp.setContextPath(contextPath);
265270
webApp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
266271
webApp.setMaxFormContentSize(maxFormContentSize);
272+
webApp.setMaxFormKeys(maxFormKeys);
267273

268274
// GZIP handler
269275
final GzipHandler gzipHandler = new GzipHandler();
@@ -366,4 +372,8 @@ public void setSessionTimeout(int sessionTimeout) {
366372
public void setMaxFormContentSize(int maxFormContentSize) {
367373
this.maxFormContentSize = maxFormContentSize;
368374
}
375+
376+
public void setMaxFormKeys(int maxFormKeys) {
377+
this.maxFormKeys = maxFormKeys;
378+
}
369379
}

‎plugins/metrics/src/main/java/org/apache/cloudstack/api/ListSystemVMsUsageHistoryCmd.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626

2727
@APICommand(name = "listSystemVmsUsageHistory", description = "Lists System VM stats", responseObject = VmMetricsStatsResponse.class,
2828
requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.18.0",
29-
authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin})
29+
authorized = {RoleType.Admin})
3030
public class ListSystemVMsUsageHistoryCmd extends BaseResourceUsageHistoryCmd {
3131

3232
/////////////////////////////////////////////////////

‎plugins/storage/volume/linstor/CHANGELOG.md

+6
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file
55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
66
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
77

8+
## [2025-01-20]
9+
10+
### Fixed
11+
12+
- Volume snapshots on zfs used the wrong dataset path to hide/unhide snapdev
13+
814
## [2024-12-13]
915

1016
### Fixed

‎plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java

+10-4
Original file line numberDiff line numberDiff line change
@@ -46,11 +46,17 @@ public final class LinstorBackupSnapshotCommandWrapper
4646
{
4747
protected static Logger LOGGER = LogManager.getLogger(LinstorBackupSnapshotCommandWrapper.class);
4848

49+
private static String zfsDatasetName(String zfsFullSnapshotUrl) {
50+
String zfsFullPath = zfsFullSnapshotUrl.substring(6);
51+
int atPos = zfsFullPath.indexOf('@');
52+
return atPos >= 0 ? zfsFullPath.substring(0, atPos) : zfsFullPath;
53+
}
54+
4955
private String zfsSnapdev(boolean hide, String zfsUrl) {
50-
Script script = new Script("/usr/bin/zfs", Duration.millis(5000));
56+
Script script = new Script("zfs", Duration.millis(5000));
5157
script.add("set");
5258
script.add("snapdev=" + (hide ? "hidden" : "visible"));
53-
script.add(zfsUrl.substring(6)); // cutting zfs://
59+
script.add(zfsDatasetName(zfsUrl)); // cutting zfs:// and @snapshotname
5460
return script.execute();
5561
}
5662

@@ -134,10 +140,10 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR
134140
LOGGER.info("Src: " + srcPath + " | " + src.getName());
135141
if (srcPath.startsWith("zfs://")) {
136142
zfsHidden = true;
137-
if (zfsSnapdev(false, srcPath) != null) {
143+
if (zfsSnapdev(false, src.getPath()) != null) {
138144
return new CopyCmdAnswer("Unable to unhide zfs snapshot device.");
139145
}
140-
srcPath = "/dev/" + srcPath.substring(6);
146+
srcPath = "/dev/zvol/" + srcPath.substring(6);
141147
}
142148

143149
secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl());

‎server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

+1
Original file line numberDiff line numberDiff line change
@@ -1413,6 +1413,7 @@ private Pair<List<Long>, Integer> searchForUserVMIdsAndCount(ListVMsCmd cmd) {
14131413
if (networkId != null || vpcId != null) {
14141414
SearchBuilder<NicVO> nicSearch = nicDao.createSearchBuilder();
14151415
nicSearch.and("networkId", nicSearch.entity().getNetworkId(), Op.EQ);
1416+
nicSearch.and("removed", nicSearch.entity().getRemoved(), Op.NULL);
14161417
if (vpcId != null) {
14171418
SearchBuilder<NetworkVO> networkSearch = networkDao.createSearchBuilder();
14181419
networkSearch.and("vpcId", networkSearch.entity().getVpcId(), Op.EQ);

‎server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java

+53-6
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.apache.cloudstack.context.CallContext;
3838
import org.apache.cloudstack.framework.config.ConfigKey;
3939
import org.apache.commons.collections.CollectionUtils;
40+
import org.apache.commons.lang3.ObjectUtils;
4041

4142
import com.cloud.agent.AgentManager;
4243
import com.cloud.agent.api.Answer;
@@ -64,12 +65,16 @@
6465
import com.cloud.service.ServiceOfferingVO;
6566
import com.cloud.service.dao.ServiceOfferingDao;
6667
import com.cloud.utils.Pair;
68+
import com.cloud.utils.StringUtils;
6769
import com.cloud.utils.Ternary;
6870
import com.cloud.utils.component.ManagerBase;
6971
import com.cloud.utils.exception.CloudRuntimeException;
72+
import com.cloud.vm.UserVmDetailVO;
7073
import com.cloud.vm.VMInstanceVO;
7174
import com.cloud.vm.VirtualMachine.State;
7275
import com.cloud.vm.VirtualMachineProfileImpl;
76+
import com.cloud.vm.VmDetailConstants;
77+
import com.cloud.vm.dao.UserVmDetailsDao;
7378
import com.cloud.vm.dao.VMInstanceDao;
7479

7580
public class RollingMaintenanceManagerImpl extends ManagerBase implements RollingMaintenanceManager {
@@ -85,6 +90,8 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin
8590
@Inject
8691
private VMInstanceDao vmInstanceDao;
8792
@Inject
93+
protected UserVmDetailsDao userVmDetailsDao;
94+
@Inject
8895
private ServiceOfferingDao serviceOfferingDao;
8996
@Inject
9097
private ClusterDetailsDao clusterDetailsDao;
@@ -619,10 +626,19 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
619626
int successfullyCheckedVmMigrations = 0;
620627
for (VMInstanceVO runningVM : vmsRunning) {
621628
boolean canMigrateVm = false;
629+
Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = getComputeResourcesCpuSpeedAndRamSize(runningVM);
630+
Integer cpu = cpuSpeedAndRamSize.first();
631+
Integer speed = cpuSpeedAndRamSize.second();
632+
Integer ramSize = cpuSpeedAndRamSize.third();
633+
if (ObjectUtils.anyNull(cpu, speed, ramSize)) {
634+
logger.warn("Cannot fetch compute resources for the VM {}, skipping it from the capacity check", runningVM);
635+
continue;
636+
}
637+
622638
ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(runningVM.getServiceOfferingId());
623639
for (Host hostInCluster : hostsInCluster) {
624640
if (!checkHostTags(hostTags, hostTagsDao.getHostTags(hostInCluster.getId()), serviceOffering.getHostTag())) {
625-
logger.debug(String.format("Host tags mismatch between %s and %s Skipping it from the capacity check", host, hostInCluster));
641+
logger.debug("Host tags mismatch between {} and {} Skipping it from the capacity check", host, hostInCluster);
626642
continue;
627643
}
628644
DeployDestination deployDestination = new DeployDestination(null, null, null, host);
@@ -632,13 +648,13 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
632648
affinityChecks = affinityChecks && affinityProcessor.check(vmProfile, deployDestination);
633649
}
634650
if (!affinityChecks) {
635-
logger.debug(String.format("Affinity check failed between %s and %s Skipping it from the capacity check", host, hostInCluster));
651+
logger.debug("Affinity check failed between {} and {} Skipping it from the capacity check", host, hostInCluster);
636652
continue;
637653
}
638654
boolean maxGuestLimit = capacityManager.checkIfHostReachMaxGuestLimit(host);
639-
boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), serviceOffering.getCpu(), serviceOffering.getSpeed());
640-
int cpuRequested = serviceOffering.getCpu() * serviceOffering.getSpeed();
641-
long ramRequested = serviceOffering.getRamSize() * 1024L * 1024L;
655+
boolean hostHasCPUCapacity = capacityManager.checkIfHostHasCpuCapability(hostInCluster.getId(), cpu, speed);
656+
int cpuRequested = cpu * speed;
657+
long ramRequested = ramSize * 1024L * 1024L;
642658
ClusterDetailsVO clusterDetailsCpuOvercommit = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
643659
ClusterDetailsVO clusterDetailsRamOvercommmt = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
644660
Float cpuOvercommitRatio = Float.parseFloat(clusterDetailsCpuOvercommit.getValue());
@@ -664,11 +680,42 @@ private Pair<Boolean, String> performCapacityChecksBeforeHostInMaintenance(Host
664680
return new Pair<>(true, "OK");
665681
}
666682

683+
protected Ternary<Integer, Integer, Integer> getComputeResourcesCpuSpeedAndRamSize(VMInstanceVO runningVM) {
684+
ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(runningVM.getServiceOfferingId());
685+
Integer cpu = serviceOffering.getCpu();
686+
Integer speed = serviceOffering.getSpeed();
687+
Integer ramSize = serviceOffering.getRamSize();
688+
if (!serviceOffering.isDynamic()) {
689+
return new Ternary<>(cpu, speed, ramSize);
690+
}
691+
692+
List<UserVmDetailVO> vmDetails = userVmDetailsDao.listDetails(runningVM.getId());
693+
if (CollectionUtils.isEmpty(vmDetails)) {
694+
return new Ternary<>(cpu, speed, ramSize);
695+
}
696+
697+
for (UserVmDetailVO vmDetail : vmDetails) {
698+
if (StringUtils.isBlank(vmDetail.getName()) || StringUtils.isBlank(vmDetail.getValue())) {
699+
continue;
700+
}
701+
702+
if (cpu == null && VmDetailConstants.CPU_NUMBER.equals(vmDetail.getName())) {
703+
cpu = Integer.valueOf(vmDetail.getValue());
704+
} else if (speed == null && VmDetailConstants.CPU_SPEED.equals(vmDetail.getName())) {
705+
speed = Integer.valueOf(vmDetail.getValue());
706+
} else if (ramSize == null && VmDetailConstants.MEMORY.equals(vmDetail.getName())) {
707+
ramSize = Integer.valueOf(vmDetail.getValue());
708+
}
709+
}
710+
711+
return new Ternary<>(cpu, speed, ramSize);
712+
}
713+
667714
/**
668715
* Check hosts tags
669716
*/
670717
private boolean checkHostTags(List<HostTagVO> hostTags, List<HostTagVO> hostInClusterTags, String offeringTag) {
671-
if (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) {
718+
if ((CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) || StringUtils.isBlank(offeringTag)) {
672719
return true;
673720
} else if ((CollectionUtils.isNotEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) ||
674721
(CollectionUtils.isEmpty(hostTags) && CollectionUtils.isNotEmpty(hostInClusterTags))) {

‎server/src/test/java/com/cloud/resource/RollingMaintenanceManagerImplTest.java

+60
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,15 @@
2323
import com.cloud.host.dao.HostDao;
2424
import com.cloud.hypervisor.Hypervisor;
2525
import com.cloud.org.Cluster;
26+
import com.cloud.service.ServiceOfferingVO;
27+
import com.cloud.service.dao.ServiceOfferingDao;
28+
import com.cloud.utils.Ternary;
2629
import com.cloud.utils.exception.CloudRuntimeException;
30+
import com.cloud.vm.UserVmDetailVO;
31+
import com.cloud.vm.VMInstanceVO;
32+
import com.cloud.vm.VmDetailConstants;
33+
import com.cloud.vm.dao.UserVmDetailsDao;
34+
2735
import org.junit.After;
2836
import org.junit.Assert;
2937
import org.junit.Before;
@@ -54,6 +62,12 @@ public class RollingMaintenanceManagerImplTest {
5462
HostVO host4;
5563
@Mock
5664
Cluster cluster;
65+
@Mock
66+
VMInstanceVO vm;
67+
@Mock
68+
ServiceOfferingDao serviceOfferingDao;
69+
@Mock
70+
UserVmDetailsDao userVmDetailsDao;
5771

5872
@Spy
5973
@InjectMocks
@@ -172,4 +186,50 @@ public void testPerformStateChecksForce() {
172186

173187
Assert.assertEquals(1, hosts.size());
174188
}
189+
190+
@Test
191+
public void testGetComputeResourcesCpuSpeedAndRamSize_ForNormalOffering() {
192+
ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class);
193+
Mockito.when(serviceOffering.isDynamic()).thenReturn(false);
194+
Mockito.when(serviceOffering.getCpu()).thenReturn(1);
195+
Mockito.when(serviceOffering.getSpeed()).thenReturn(500);
196+
Mockito.when(serviceOffering.getRamSize()).thenReturn(512);
197+
198+
Mockito.when(vm.getServiceOfferingId()).thenReturn(1L);
199+
Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering);
200+
201+
Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = manager.getComputeResourcesCpuSpeedAndRamSize(vm);
202+
203+
Assert.assertEquals(1, cpuSpeedAndRamSize.first().intValue());
204+
Assert.assertEquals(500, cpuSpeedAndRamSize.second().intValue());
205+
Assert.assertEquals(512, cpuSpeedAndRamSize.third().intValue());
206+
}
207+
208+
@Test
209+
public void testGetComputeResourcesCpuSpeedAndRamSize_ForCustomOffering() {
210+
ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class);
211+
Mockito.when(serviceOffering.isDynamic()).thenReturn(true);
212+
Mockito.when(serviceOffering.getCpu()).thenReturn(null);
213+
Mockito.when(serviceOffering.getSpeed()).thenReturn(null);
214+
Mockito.when(serviceOffering.getRamSize()).thenReturn(null);
215+
216+
List<UserVmDetailVO> vmDetails = new ArrayList<>();
217+
UserVmDetailVO cpuDetail = new UserVmDetailVO(1L, VmDetailConstants.CPU_NUMBER, "2", false);
218+
vmDetails.add(cpuDetail);
219+
UserVmDetailVO speedDetail = new UserVmDetailVO(1L, VmDetailConstants.CPU_SPEED, "1000", false);
220+
vmDetails.add(speedDetail);
221+
UserVmDetailVO ramSizeDetail = new UserVmDetailVO(1L, VmDetailConstants.MEMORY, "1024", false);
222+
vmDetails.add(ramSizeDetail);
223+
224+
Mockito.when(vm.getId()).thenReturn(1L);
225+
Mockito.when(vm.getServiceOfferingId()).thenReturn(1L);
226+
Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering);
227+
Mockito.when(userVmDetailsDao.listDetails(1L)).thenReturn(vmDetails);
228+
229+
Ternary<Integer, Integer, Integer> cpuSpeedAndRamSize = manager.getComputeResourcesCpuSpeedAndRamSize(vm);
230+
231+
Assert.assertEquals(2, cpuSpeedAndRamSize.first().intValue());
232+
Assert.assertEquals(1000, cpuSpeedAndRamSize.second().intValue());
233+
Assert.assertEquals(1024, cpuSpeedAndRamSize.third().intValue());
234+
}
175235
}

0 commit comments

Comments
 (0)