Support multi-scope configuration settings (#10300)

This PR introduces the concept of multi-scope configuration settings. In addition to the Global level, currently all configurations can be set at a single scope level.
It will be useful if a configuration can be set at multiple scopes. For example, a configuration set at the domain level
will apply for all accounts, but it can be set for an account as well. In which case the account level setting will override the domain level setting.

This is done by changing the column `scope` of table `configuration` from string (single scope) to bitmask (multiple scopes).

```
public enum Scope {
    Global(null, 1),
    Zone(Global, 1 << 1),
    Cluster(Zone, 1 << 2),
    StoragePool(Cluster, 1 << 3),
    ManagementServer(Global, 1 << 4),
    ImageStore(Zone, 1 << 5),
    Domain(Global, 1 << 6),
    Account(Domain, 1 << 7);
```
Each scope is also assigned a parent scope. When a configuration for a given scope is not defined but is available for multiple scope types, the value will be retrieved from the parent scope. If there is no parent scope or if the configuration is defined for a single scope only, the value will fall back to the global level.

Hierarchy for different scopes is defined as below :
- Global
    - Zone
        - Cluster
            - Storage Pool
        - Image Store
    - Management Server
    - Domain
        - Account

This PR also updates the scope of the following configurations (Storage Pool scope is added in addition to the existing Zone scope):
- pool.storage.allocated.capacity.disablethreshold
- pool.storage.allocated.resize.capacity.disablethreshold
- pool.storage.capacity.disablethreshold

Doc PR : apache/cloudstack-documentation#476

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
Co-authored-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

Author: abh1sar

engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.capacity;
 
+import java.util.List;
+
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
@@ -67,7 +69,7 @@ public interface CapacityManager {
                     "0.85",
                     "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.",
                     true,
-                    ConfigKey.Scope.Zone);
+                    List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone));
     static final ConfigKey<Double> StorageOverprovisioningFactor =
             new ConfigKey<>(
                     "Storage",
@@ -85,7 +87,7 @@ public interface CapacityManager {
                     "0.85",
                     "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.",
                     true,
-                    ConfigKey.Scope.Zone);
+                    List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone));
     static final ConfigKey<Boolean> StorageOperationsExcludeCluster =
             new ConfigKey<>(
                     Boolean.class,
@@ -125,7 +127,7 @@ public interface CapacityManager {
                     "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for volume resize. " +
                             "This is applicable only when volume.resize.allowed.beyond.allocation is set to true.",
                     true,
-                    ConfigKey.Scope.Zone);
+                    List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone));
 
     ConfigKey<Integer> CapacityCalculateWorkers = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class,
             "capacity.calculate.workers", "1",

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -214,7 +214,7 @@ public interface StorageManager extends StorageService {
     ConfigKey<Boolean> AllowVolumeReSizeBeyondAllocation = new ConfigKey<Boolean>("Advanced", Boolean.class, "volume.resize.allowed.beyond.allocation", "false",
             "Determines whether volume size can exceed the pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold) " +
                     "when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)",
-            true, ConfigKey.Scope.Zone);
+            true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone));
 
     ConfigKey<Integer> StoragePoolHostConnectWorkers = new ConfigKey<>("Storage", Integer.class,
             "storage.pool.host.connect.workers", "1",

engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java

@@ -22,11 +22,16 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import javax.inject.Inject;
+
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.ConfigKey.Scope;
 import org.apache.cloudstack.framework.config.ScopedConfigStorage;
 import org.apache.commons.collections.CollectionUtils;
 
+import com.cloud.dc.dao.ClusterDao;
+import com.cloud.org.Cluster;
+import com.cloud.utils.Pair;
 import com.cloud.utils.crypt.DBEncryptionUtil;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
@@ -35,6 +40,9 @@
 
 public class ClusterDetailsDaoImpl extends ResourceDetailsDaoBase<ClusterDetailsVO> implements ClusterDetailsDao, ScopedConfigStorage {
 
+    @Inject
+    ClusterDao clusterDao;
+
     protected final SearchBuilder<ClusterDetailsVO> ClusterSearch;
     protected final SearchBuilder<ClusterDetailsVO> DetailSearch;
 
@@ -186,4 +194,13 @@ private String getCpuMemoryOvercommitRatio(String name) {
 
         return name;
     }
+
+    @Override
+    public Pair<Scope, Long> getParentScope(long id) {
+        Cluster cluster = clusterDao.findById(id);
+        if (cluster == null) {
+            return null;
+        }
+        return new Pair<>(getScope().getParent(), cluster.getDataCenterId());
+    }
 }

engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java

@@ -30,6 +30,8 @@
 import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
+import com.cloud.utils.Pair;
+
 public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase<StoragePoolDetailVO> implements StoragePoolDetailsDao, ScopedConfigStorage {
 
     @Inject
@@ -57,4 +59,17 @@ public void addDetail(long resourceId, String key, String value, boolean display
         }
         super.addDetail(new StoragePoolDetailVO(resourceId, key, value, display));
     }
+
+    @Override
+    public Pair<Scope, Long> getParentScope(long id) {
+        StoragePoolVO pool = _storagePoolDao.findById(id);
+        if (pool != null) {
+            if (pool.getClusterId() != null) {
+                return new Pair<>(getScope().getParent(), pool.getClusterId());
+            } else {
+                return new Pair<>(ConfigKey.Scope.Zone, pool.getDataCenterId());
+            }
+        }
+        return null;
+    }
 }

engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java

@@ -54,7 +54,7 @@ public ConfigurationGroupsAggregator() {
 
     public void updateConfigurationGroups() {
         LOG.debug("Updating configuration groups");
-        List<ConfigurationVO> configs =  configDao.listAllIncludingRemoved();
+        List<ConfigurationVO> configs =  configDao.searchPartialConfigurations();
         if (CollectionUtils.isEmpty(configs)) {
             return;
         }

engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java

@@ -87,6 +87,36 @@ public boolean columnExists(Connection conn, String tableName, String columnName
         return columnExists;
     }
 
+    public String getColumnType(Connection conn, String tableName, String columnName) {
+        try (PreparedStatement pstmt = conn.prepareStatement(String.format("DESCRIBE %s %s", tableName, columnName));){
+            ResultSet rs = pstmt.executeQuery();
+            if (rs.next()) {
+                return rs.getString("Type");
+            }
+        } catch (SQLException e) {
+            logger.warn("Type for column {} can not be retrieved in {} ignoring exception: {}", columnName, tableName, e.getMessage());
+        }
+        return null;
+    }
+
+    public void addColumn(Connection conn, String tableName, String columnName, String columnDefinition) {
+        try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition));){
+            pstmt.executeUpdate();
+            logger.debug("Column {} is added successfully from the table {}", columnName, tableName);
+        } catch (SQLException e) {
+            logger.warn("Unable to add column {} to table {} due to exception", columnName, tableName, e);
+        }
+    }
+
+    public void changeColumn(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) {
+        try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s CHANGE COLUMN %s %s %s", tableName, oldColumnName, newColumnName, columnDefinition));){
+            pstmt.executeUpdate();
+            logger.debug("Column {} is changed successfully to {} from the table {}", oldColumnName, newColumnName, tableName);
+        } catch (SQLException e) {
+            logger.warn("Unable to add column {} to {} from the table {} due to exception", oldColumnName, newColumnName, tableName, e);
+        }
+    }
+
     public String generateIndexName(String tableName, String... columnName) {
         return String.format("i_%s__%s", tableName, StringUtils.join(columnName, "__"));
     }

engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java

@@ -58,4 +58,20 @@ public static void dropTableColumnsIfExist(Connection conn, String tableName, Li
         }
     }
 
+    public static String getTableColumnType(Connection conn, String tableName, String columnName) {
+        return dao.getColumnType(conn, tableName, columnName);
+    }
+
+    public static void addTableColumnIfNotExist(Connection conn, String tableName, String columnName, String columnDefinition) {
+        if (!dao.columnExists(conn, tableName, columnName)) {
+            dao.addColumn(conn, tableName, columnName, columnDefinition);
+        }
+    }
+
+    public static void changeTableColumnIfNotExist(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) {
+        if (dao.columnExists(conn, tableName, oldColumnName)) {
+            dao.changeColumn(conn, tableName, oldColumnName, newColumnName, columnDefinition);
+        }
+    }
+
 }

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java

@@ -17,10 +17,16 @@
 package com.cloud.upgrade.dao;
 
 import com.cloud.upgrade.SystemVmTemplateRegistration;
+import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 import java.io.InputStream;
 import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+
+import org.apache.cloudstack.framework.config.ConfigKey;
 
 public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
     private SystemVmTemplateRegistration systemVmTemplateRegistration;
@@ -53,6 +59,7 @@ public InputStream[] getPrepareScripts() {
 
     @Override
     public void performDataMigration(Connection conn) {
+        migrateConfigurationScopeToBitmask(conn);
     }
 
     @Override
@@ -80,4 +87,35 @@ public void updateSystemVmTemplates(Connection conn) {
             throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
         }
     }
+
+    protected void migrateConfigurationScopeToBitmask(Connection conn) {
+        String scopeDataType = DbUpgradeUtils.getTableColumnType(conn, "configuration", "scope");
+        logger.info("Data type of the column scope of table configuration is {}", scopeDataType);
+        if (!"varchar(255)".equals(scopeDataType)) {
+            return;
+        }
+        DbUpgradeUtils.addTableColumnIfNotExist(conn, "configuration", "new_scope", "BIGINT DEFAULT 0");
+        migrateExistingConfigurationScopeValues(conn);
+        DbUpgradeUtils.dropTableColumnsIfExist(conn, "configuration", List.of("scope"));
+        DbUpgradeUtils.changeTableColumnIfNotExist(conn, "configuration", "new_scope", "scope", "BIGINT NOT NULL DEFAULT 0 COMMENT 'Bitmask for scope(s) of this parameter'");
+    }
+
+    protected void migrateExistingConfigurationScopeValues(Connection conn) {
+        StringBuilder sql = new StringBuilder("UPDATE configuration\n" +
+                "SET new_scope = " +
+                "    CASE ");
+        for (ConfigKey.Scope scope : ConfigKey.Scope.values()) {
+            sql.append("        WHEN scope = '").append(scope.name()).append("' THEN ").append(scope.getBitValue()).append(" ");
+        }
+        sql.append("        ELSE 0 " +
+                "    END " +
+                "WHERE scope IS NOT NULL;");
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql.toString())) {
+            pstmt.executeUpdate();
+        } catch (SQLException e) {
+            logger.error("Failed to migrate existing configuration scope values to bitmask", e);
+            throw new CloudRuntimeException(String.format("Failed to migrate existing configuration scope values to bitmask due to: %s", e.getMessage()));
+        }
+    }
 }

engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java

@@ -33,6 +33,7 @@
 import com.cloud.domain.dao.DomainDao;
 import com.cloud.domain.dao.DomainDetailsDao;
 import com.cloud.user.dao.AccountDao;
+import com.cloud.utils.Pair;
 import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
@@ -156,4 +157,13 @@ public String getConfigValue(long id, String key) {
         }
         return value;
     }
+
+    @Override
+    public Pair<Scope, Long> getParentScope(long id) {
+        Account account = _accountDao.findById(id);
+        if (account == null) {
+            return null;
+        }
+        return new Pair<>(getScope().getParent(), account.getDomainId());
+    }
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java

@@ -20,13 +20,17 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.ConfigKey.Scope;
 import org.apache.cloudstack.framework.config.ScopedConfigStorage;
 import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase;
 import org.springframework.stereotype.Component;
 
+import com.cloud.storage.ImageStore;
+import com.cloud.utils.Pair;
 import com.cloud.utils.crypt.DBEncryptionUtil;
 import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchBuilder;
@@ -36,6 +40,9 @@
 
 @Component
 public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase<ImageStoreDetailVO> implements ImageStoreDetailsDao, ScopedConfigStorage {
+    @Inject
+    ImageStoreDao imageStoreDao;
+
     protected final SearchBuilder<ImageStoreDetailVO> storeSearch;
 
     public ImageStoreDetailsDaoImpl() {
@@ -113,10 +120,20 @@ public String getConfigValue(long id, String key) {
     public String getConfigValue(long id, ConfigKey<?> key) {
         ImageStoreDetailVO vo = findDetail(id, key.key());
         return vo == null ? null : getActualValue(vo);
-        }
+    }
 
     @Override
     public void addDetail(long resourceId, String key, String value, boolean display) {
         super.addDetail(new ImageStoreDetailVO(resourceId, key, value, display));
     }
+
+    @Override
+    public Pair<Scope, Long> getParentScope(long id) {
+        ImageStore store = imageStoreDao.findById(id);
+        if (store == null) {
+            return null;
+        }
+        return new Pair<>(getScope().getParent(), store.getDataCenterId());
+    }
+
 }

engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java

@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade;
+
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao;
+import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao;
+import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO;
+import org.apache.cloudstack.framework.config.impl.ConfigurationVO;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ConfigurationGroupsAggregatorTest {
+    @InjectMocks
+    private ConfigurationGroupsAggregator configurationGroupsAggregator = new ConfigurationGroupsAggregator();
+
+    @Mock
+    private ConfigurationDao configDao;
+
+    @Mock
+    private ConfigurationGroupDao configGroupDao;
+
+    @Mock
+    private ConfigurationSubGroupDao configSubGroupDao;
+
+    @Mock
+    private Logger logger;
+
+    @Test
+    public void testUpdateConfigurationGroups() {
+        ConfigurationVO config = new ConfigurationVO("Advanced", "DEFAULT", "management-server",
+                "test.config.name", null, "description");
+        config.setGroupId(1L);
+        config.setSubGroupId(1L);
+
+        when(configDao.searchPartialConfigurations()).thenReturn(Collections.singletonList(config));
+
+        ConfigurationSubGroupVO configSubGroup = Mockito.mock(ConfigurationSubGroupVO.class);
+        when(configSubGroupDao.findByName("name")).thenReturn(configSubGroup);
+        Mockito.when(configSubGroup.getId()).thenReturn(10L);
+        Mockito.when(configSubGroup.getGroupId()).thenReturn(5L);
+
+        configurationGroupsAggregator.updateConfigurationGroups();
+
+        Assert.assertEquals(Long.valueOf(5), config.getGroupId());
+        Assert.assertEquals(Long.valueOf(10), config.getSubGroupId());
+        Mockito.verify(configDao, Mockito.times(1)).persist(config);
+        Mockito.verify(logger, Mockito.times(1)).debug("Updating configuration groups");
+        Mockito.verify(logger, Mockito.times(1)).debug("Successfully updated configuration groups.");
+    }
+}