Skip to content

Commit e87c6cf

Browse files
vishesh92DaanHoogland
vishesh92
and
DaanHoogland
authored
Fix resource count discrepancies (#8302)
* Fix resource count discrepancies * Fixup while removing vm * Fix discrepancies when starting VMs * Fixup tests * Fix failing tests * Don't take lock when amount is negative --------- Co-authored-by: dahn <daan@onecht.net>
1 parent 6dc3d06 commit e87c6cf

File tree

18 files changed

+398
-154
lines changed

18 files changed

+398
-154
lines changed

‎api/src/main/java/org/apache/cloudstack/user/ResourceReservation.java

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434

3535
Resource.ResourceType getResourceType();
3636

37+
Long getResourceId();
38+
3739
String getTag();
3840

3941
Long getReservedAmount();

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

+33-20
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Licensed to the Apacohe Software Foundation (ASF) under one
1+
// Licensed to the Apache Software Foundation (ASF) under one
22
// or more contributor license agreements. See the NOTICE file
33
// distributed with this work for additional information
44
// regarding copyright ownership. The ASF licenses this file
@@ -49,6 +49,7 @@
4949
import javax.naming.ConfigurationException;
5050
import javax.persistence.EntityExistsException;
5151

52+
import com.cloud.configuration.Resource;
5253
import com.cloud.domain.Domain;
5354
import com.cloud.domain.dao.DomainDao;
5455
import com.cloud.network.vpc.VpcVO;
@@ -87,6 +88,7 @@
8788
import org.apache.cloudstack.framework.messagebus.MessageHandler;
8889
import org.apache.cloudstack.jobs.JobInfo;
8990
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
91+
import org.apache.cloudstack.reservation.dao.ReservationDao;
9092
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
9193
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
9294
import org.apache.cloudstack.storage.to.VolumeObjectTO;
@@ -296,6 +298,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
296298
@Inject
297299
private VMInstanceDao _vmDao;
298300
@Inject
301+
private ReservationDao _reservationDao;
302+
@Inject
299303
private ServiceOfferingDao _offeringDao;
300304
@Inject
301305
private DiskOfferingDao _diskOfferingDao;
@@ -914,7 +918,7 @@ protected boolean checkWorkItems(final VMInstanceVO vm, final State state) throw
914918

915919
@DB
916920
protected Ternary<VMInstanceVO, ReservationContext, ItWorkVO> changeToStartState(final VirtualMachineGuru vmGuru, final VMInstanceVO vm, final User caller,
917-
final Account account) throws ConcurrentOperationException {
921+
final Account account, Account owner, ServiceOfferingVO offering, VirtualMachineTemplate template) throws ConcurrentOperationException {
918922
final long vmId = vm.getId();
919923

920924
ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Starting, vm.getType(), vm.getId());
@@ -934,6 +938,9 @@ public Ternary<VMInstanceVO, ReservationContext, ItWorkVO> doInTransaction(final
934938
if (logger.isDebugEnabled()) {
935939
logger.debug("Successfully transitioned to start state for " + vm + " reservation id = " + work.getId());
936940
}
941+
if (VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) {
942+
_resourceLimitMgr.incrementVmResourceCount(owner.getAccountId(), vm.isDisplay(), offering, template);
943+
}
937944
return new Ternary<>(vm, context, work);
938945
}
939946

@@ -1126,7 +1133,10 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
11261133

11271134
final VirtualMachineGuru vmGuru = getVmGuru(vm);
11281135

1129-
final Ternary<VMInstanceVO, ReservationContext, ItWorkVO> start = changeToStartState(vmGuru, vm, caller, account);
1136+
final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
1137+
final ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
1138+
final VirtualMachineTemplate template = _entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, vm.getTemplateId());
1139+
final Ternary<VMInstanceVO, ReservationContext, ItWorkVO> start = changeToStartState(vmGuru, vm, caller, account, owner, offering, template);
11301140
if (start == null) {
11311141
return;
11321142
}
@@ -1136,8 +1146,6 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
11361146
ItWorkVO work = start.third();
11371147

11381148
VMInstanceVO startedVm = null;
1139-
final ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
1140-
final VirtualMachineTemplate template = _entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, vm.getTemplateId());
11411149

11421150
DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterId(), vm.getPodIdToDeployIn(), null, null, null, null, ctx);
11431151
if (planToDeploy != null && planToDeploy.getDataCenterId() != 0) {
@@ -1150,12 +1158,6 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
11501158

11511159
final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType());
11521160

1153-
// check resource count if ResourceCountRunningVMsonly.value() = true
1154-
final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
1155-
if (VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) {
1156-
_resourceLimitMgr.incrementVmResourceCount(owner.getAccountId(), vm.isDisplay(), offering, template);
1157-
}
1158-
11591161
boolean canRetry = true;
11601162
ExcludeList avoids = null;
11611163
try {
@@ -2277,16 +2279,21 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl
22772279
_workDao.update(work.getId(), work);
22782280
}
22792281

2280-
boolean result = stateTransitTo(vm, Event.OperationSucceeded, null);
2281-
if (result) {
2282-
vm.setPowerState(PowerState.PowerOff);
2283-
_vmDao.update(vm.getId(), vm);
2284-
if (VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) {
2285-
ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
2286-
VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId());
2287-
_resourceLimitMgr.decrementVmResourceCount(vm.getAccountId(), vm.isDisplay(), offering, template);
2282+
boolean result = Transaction.execute(new TransactionCallbackWithException<Boolean, NoTransitionException>() {
2283+
@Override
2284+
public Boolean doInTransaction(TransactionStatus status) throws NoTransitionException {
2285+
boolean result = stateTransitTo(vm, Event.OperationSucceeded, null);
2286+
2287+
if (result && VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) {
2288+
ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
2289+
VMTemplateVO template = _templateDao.findByIdIncludingRemoved(vm.getTemplateId());
2290+
_resourceLimitMgr.decrementVmResourceCount(vm.getAccountId(), vm.isDisplay(), offering, template);
2291+
}
2292+
return result;
22882293
}
2289-
} else {
2294+
});
2295+
2296+
if (!result) {
22902297
throw new CloudRuntimeException("unable to stop " + vm);
22912298
}
22922299
} catch (final NoTransitionException e) {
@@ -2319,6 +2326,12 @@ public boolean stateTransitTo(final VirtualMachine vm1, final VirtualMachine.Eve
23192326
vm.setLastHostId(vm.getHostId());
23202327
}
23212328
}
2329+
2330+
if (e.equals(VirtualMachine.Event.DestroyRequested) || e.equals(VirtualMachine.Event.ExpungeOperation)) {
2331+
_reservationDao.setResourceId(Resource.ResourceType.user_vm, null);
2332+
_reservationDao.setResourceId(Resource.ResourceType.cpu, null);
2333+
_reservationDao.setResourceId(Resource.ResourceType.memory, null);
2334+
}
23222335
return _stateMachine.transitTo(vm, e, new Pair<>(vm.getHostId(), hostId), _vmDao);
23232336
}
23242337

‎engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -1006,6 +1006,7 @@ public void checkIfVmNetworkDetailsReturnedIsCorrect() {
10061006
public void testOrchestrateStartNonNullPodId() throws Exception {
10071007
VMInstanceVO vmInstance = new VMInstanceVO();
10081008
ReflectionTestUtils.setField(vmInstance, "id", 1L);
1009+
ReflectionTestUtils.setField(vmInstance, "accountId", 1L);
10091010
ReflectionTestUtils.setField(vmInstance, "uuid", "vm-uuid");
10101011
ReflectionTestUtils.setField(vmInstance, "serviceOfferingId", 2L);
10111012
ReflectionTestUtils.setField(vmInstance, "instanceName", "myVm");
@@ -1019,6 +1020,7 @@ public void testOrchestrateStartNonNullPodId() throws Exception {
10191020
User user = mock(User.class);
10201021

10211022
Account account = mock(Account.class);
1023+
Account owner = mock(Account.class);
10221024

10231025
ReservationContext ctx = mock(ReservationContext.class);
10241026

@@ -1042,12 +1044,13 @@ public void testOrchestrateStartNonNullPodId() throws Exception {
10421044
doReturn(vmGuru).when(virtualMachineManagerImpl).getVmGuru(vmInstance);
10431045

10441046
Ternary<VMInstanceVO, ReservationContext, ItWorkVO> start = new Ternary<>(vmInstance, ctx, work);
1045-
Mockito.doReturn(start).when(virtualMachineManagerImpl).changeToStartState(vmGuru, vmInstance, user, account);
1047+
Mockito.doReturn(start).when(virtualMachineManagerImpl).changeToStartState(vmGuru, vmInstance, user, account, owner, serviceOffering, template);
10461048

10471049
when(ctx.getJournal()).thenReturn(Mockito.mock(Journal.class));
10481050

10491051
when(serviceOfferingDaoMock.findById(vmInstance.getId(), vmInstance.getServiceOfferingId())).thenReturn(serviceOffering);
10501052

1053+
when(_entityMgr.findById(Account.class, vmInstance.getAccountId())).thenReturn(owner);
10511054
when(_entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, vmInstance.getTemplateId())).thenReturn(template);
10521055

10531056
Host destHost = mock(Host.class);
@@ -1099,6 +1102,7 @@ public void testOrchestrateStartNonNullPodId() throws Exception {
10991102
public void testOrchestrateStartNullPodId() throws Exception {
11001103
VMInstanceVO vmInstance = new VMInstanceVO();
11011104
ReflectionTestUtils.setField(vmInstance, "id", 1L);
1105+
ReflectionTestUtils.setField(vmInstance, "accountId", 1L);
11021106
ReflectionTestUtils.setField(vmInstance, "uuid", "vm-uuid");
11031107
ReflectionTestUtils.setField(vmInstance, "serviceOfferingId", 2L);
11041108
ReflectionTestUtils.setField(vmInstance, "instanceName", "myVm");
@@ -1112,6 +1116,7 @@ public void testOrchestrateStartNullPodId() throws Exception {
11121116
User user = mock(User.class);
11131117

11141118
Account account = mock(Account.class);
1119+
Account owner = mock(Account.class);
11151120

11161121
ReservationContext ctx = mock(ReservationContext.class);
11171122

@@ -1135,12 +1140,13 @@ public void testOrchestrateStartNullPodId() throws Exception {
11351140
doReturn(vmGuru).when(virtualMachineManagerImpl).getVmGuru(vmInstance);
11361141

11371142
Ternary<VMInstanceVO, ReservationContext, ItWorkVO> start = new Ternary<>(vmInstance, ctx, work);
1138-
Mockito.doReturn(start).when(virtualMachineManagerImpl).changeToStartState(vmGuru, vmInstance, user, account);
1143+
Mockito.doReturn(start).when(virtualMachineManagerImpl).changeToStartState(vmGuru, vmInstance, user, account, owner, serviceOffering, template);
11391144

11401145
when(ctx.getJournal()).thenReturn(Mockito.mock(Journal.class));
11411146

11421147
when(serviceOfferingDaoMock.findById(vmInstance.getId(), vmInstance.getServiceOfferingId())).thenReturn(serviceOffering);
11431148

1149+
when(_entityMgr.findById(Account.class, vmInstance.getAccountId())).thenReturn(owner);
11441150
when(_entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, vmInstance.getTemplateId())).thenReturn(template);
11451151

11461152
Host destHost = mock(Host.class);

‎engine/schema/src/main/java/com/cloud/storage/VolumeVO.java

+1
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ public class VolumeVO implements Volume {
182182
@Column(name = "encrypt_format")
183183
private String encryptFormat;
184184

185+
185186
// Real Constructor
186187
public VolumeVO(Type type, String name, long dcId, long domainId,
187188
long accountId, long diskOfferingId, Storage.ProvisioningType provisioningType, long size,

‎engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java

+34-1
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,15 @@
2323
import java.util.Collections;
2424
import java.util.Date;
2525
import java.util.List;
26+
import java.util.stream.Collectors;
2627

2728
import javax.inject.Inject;
2829

30+
import com.cloud.configuration.Resource;
31+
import com.cloud.utils.db.Transaction;
32+
import com.cloud.utils.db.TransactionCallback;
33+
import org.apache.cloudstack.reservation.ReservationVO;
34+
import org.apache.cloudstack.reservation.dao.ReservationDao;
2935
import org.apache.commons.collections.CollectionUtils;
3036
import org.springframework.stereotype.Component;
3137

@@ -71,6 +77,8 @@ public class VolumeDaoImpl extends GenericDaoBase<VolumeVO, Long> implements Vol
7177
protected GenericSearchBuilder<VolumeVO, SumCount> secondaryStorageSearch;
7278
private final SearchBuilder<VolumeVO> poolAndPathSearch;
7379
@Inject
80+
ReservationDao reservationDao;
81+
@Inject
7482
ResourceTagDao _tagsDao;
7583

7684
protected static final String SELECT_VM_SQL = "SELECT DISTINCT instance_id from volumes v where v.host_id = ? and v.mirror_state = ?";
@@ -443,6 +451,7 @@ public VolumeDaoImpl() {
443451
CountByAccount.and("account", CountByAccount.entity().getAccountId(), SearchCriteria.Op.EQ);
444452
CountByAccount.and("state", CountByAccount.entity().getState(), SearchCriteria.Op.NIN);
445453
CountByAccount.and("displayVolume", CountByAccount.entity().isDisplayVolume(), Op.EQ);
454+
CountByAccount.and("idNIN", CountByAccount.entity().getId(), Op.NIN);
446455
CountByAccount.done();
447456

448457
primaryStorageSearch = createSearchBuilder(SumCount.class);
@@ -454,6 +463,7 @@ public VolumeDaoImpl() {
454463
primaryStorageSearch.and("displayVolume", primaryStorageSearch.entity().isDisplayVolume(), Op.EQ);
455464
primaryStorageSearch.and("isRemoved", primaryStorageSearch.entity().getRemoved(), Op.NULL);
456465
primaryStorageSearch.and("NotCountStates", primaryStorageSearch.entity().getState(), Op.NIN);
466+
primaryStorageSearch.and("idNIN", primaryStorageSearch.entity().getId(), Op.NIN);
457467
primaryStorageSearch.done();
458468

459469
primaryStorageSearch2 = createSearchBuilder(SumCount.class);
@@ -468,6 +478,7 @@ public VolumeDaoImpl() {
468478
primaryStorageSearch2.and("displayVolume", primaryStorageSearch2.entity().isDisplayVolume(), Op.EQ);
469479
primaryStorageSearch2.and("isRemoved", primaryStorageSearch2.entity().getRemoved(), Op.NULL);
470480
primaryStorageSearch2.and("NotCountStates", primaryStorageSearch2.entity().getState(), Op.NIN);
481+
primaryStorageSearch2.and("idNIN", primaryStorageSearch2.entity().getId(), Op.NIN);
471482
primaryStorageSearch2.done();
472483

473484
secondaryStorageSearch = createSearchBuilder(SumCount.class);
@@ -506,15 +517,24 @@ public Pair<Long, Long> getCountAndTotalByPool(long poolId) {
506517

507518
@Override
508519
public Long countAllocatedVolumesForAccount(long accountId) {
520+
List<ReservationVO> reservations = reservationDao.getReservationsForAccount(accountId, Resource.ResourceType.volume, null);
521+
List<Long> reservedResourceIds = reservations.stream().filter(reservation -> reservation.getReservedAmount() > 0).map(ReservationVO::getResourceId).collect(Collectors.toList());
522+
509523
SearchCriteria<Long> sc = CountByAccount.create();
510524
sc.setParameters("account", accountId);
511-
sc.setParameters("state", Volume.State.Destroy, Volume.State.Expunged);
525+
sc.setParameters("state", State.Destroy, State.Expunged);
512526
sc.setParameters("displayVolume", 1);
527+
if (CollectionUtils.isNotEmpty(reservedResourceIds)) {
528+
sc.setParameters("idNIN", reservedResourceIds.toArray());
529+
}
513530
return customSearch(sc, null).get(0);
514531
}
515532

516533
@Override
517534
public long primaryStorageUsedForAccount(long accountId, List<Long> virtualRouters) {
535+
List<ReservationVO> reservations = reservationDao.getReservationsForAccount(accountId, Resource.ResourceType.volume, null);
536+
List<Long> reservedResourceIds = reservations.stream().filter(reservation -> reservation.getReservedAmount() > 0).map(ReservationVO::getResourceId).collect(Collectors.toList());
537+
518538
SearchCriteria<SumCount> sc;
519539
if (!virtualRouters.isEmpty()) {
520540
sc = primaryStorageSearch2.create();
@@ -526,6 +546,9 @@ public long primaryStorageUsedForAccount(long accountId, List<Long> virtualRoute
526546
sc.setParameters("states", State.Allocated);
527547
sc.setParameters("NotCountStates", State.Destroy, State.Expunged);
528548
sc.setParameters("displayVolume", 1);
549+
if (CollectionUtils.isNotEmpty(reservedResourceIds)) {
550+
sc.setParameters("idNIN", reservedResourceIds.toArray());
551+
}
529552
List<SumCount> storageSpace = customSearch(sc, null);
530553
if (storageSpace != null) {
531554
return storageSpace.get(0).sum;
@@ -863,4 +886,14 @@ public List<VolumeVO> listAllocatedVolumesForAccountDiskOfferingIdsAndNotForVms(
863886
}
864887
return listBy(sc);
865888
}
889+
890+
@Override
891+
public VolumeVO persist(VolumeVO entity) {
892+
return Transaction.execute((TransactionCallback<VolumeVO>) status -> {
893+
VolumeVO volume = super.persist(entity);
894+
reservationDao.setResourceId(Resource.ResourceType.volume, volume.getId());
895+
reservationDao.setResourceId(Resource.ResourceType.primary_storage, volume.getId());
896+
return volume;
897+
});
898+
}
866899
}

0 commit comments

Comments
 (0)