From dc0c47439cba76e9b7b332762fe6416dc926ffe2 Mon Sep 17 00:00:00 2001
From: John Payne <89417863+jpayne3506@users.noreply.github.com>
Date: Tue, 9 Jul 2024 13:44:00 -0700
Subject: [PATCH] ci: Adjust VM usage for CNI Release Test Pipeline (#2837)

ci: Add variables for sku region control
---
 .../cilium-overlay-load-test-template.yaml    | 15 +++++++++-
 .../restart-node-template.yaml                |  9 ++++--
 .../lsg/lsg-cni-intergration-template.yaml    |  1 +
 .pipelines/cni/pipeline.yaml                  | 16 ++++++++--
 .../cni/singletenancy/cniv1-template.yaml     | 22 ++++++++++++--
 .../cni/singletenancy/cniv2-template.yaml     | 30 +++++++++++++++++--
 6 files changed, 83 insertions(+), 10 deletions(-)

diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
index 9982db114f..9fe1fbda74 100644
--- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
+++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml
@@ -19,6 +19,12 @@ stages:
   - stage: create_${{ parameters.name }}
     condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cilium') , contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'linux'), contains(variables.CONTROL_OS, 'all') ) ) )
     variables:
+      ${{ if contains(parameters.clusterName, 'rdma') }}:
+        location: $(LOCATION_RDMA)
+      ${{ elseif eq(parameters.arch, 'arm64') }}:
+        location: $(LOCATION_ARM64)
+      ${{ else }}:
+        location: $(LOCATION_AMD64)
       commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
     dependsOn:
       - setup
@@ -34,7 +40,7 @@ stages:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               nodeCount: ${{ parameters.nodeCount }}
               vmSize: ${{ parameters.vmSize }}
-              region: $(LOCATION)
+              region: $(location)
 
 # Conditions for below E2E test scenarios confirm that:
 # Pipeline has not been canceled and that the previous job has reports anything other than failure(Succeeded, SuccededWithIssues, Skipped). Previous job is declared by dependsOn:
@@ -44,6 +50,12 @@ stages:
     variables:
       commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
       cnsVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.cnsVersion'] ]
+      ${{ if contains(parameters.clusterName, 'rdma') }}:
+        location: $(LOCATION_RDMA)
+      ${{ elseif eq(parameters.arch, 'arm64') }}:
+        location: $(LOCATION_ARM64)
+      ${{ else }}:
+        location: $(LOCATION_AMD64)
     pool:
       name: "$(BUILD_POOL_NAME_DEFAULT)"
     dependsOn:
@@ -162,6 +174,7 @@ stages:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               os: ${{ parameters.os }}
               cni: cilium
+              region: $(location)
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
diff --git a/.pipelines/cni/load-test-templates/restart-node-template.yaml b/.pipelines/cni/load-test-templates/restart-node-template.yaml
index ff54fc8215..e1a54cb49b 100644
--- a/.pipelines/cni/load-test-templates/restart-node-template.yaml
+++ b/.pipelines/cni/load-test-templates/restart-node-template.yaml
@@ -3,6 +3,7 @@ parameters:
   os: "linux"
   cni: ""
   jobName: "restart_nodes"
+  region: ""
 
 steps:
   - task: AzureCLI@1
@@ -13,12 +14,14 @@ steps:
       addSpnToEnvironment: true
       inlineScript: |
         clusterName=${{ parameters.clusterName }}
+        region=${{ parameters.region }}
+
         make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
-        make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
+        make -C ./hack/aks azcfg AZCLI=az REGION=${region}
 
         echo "Restarting the nodes"
-        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(LOCATION) --query "[].name" -o tsv); do
-          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(LOCATION) VMSS_NAME=$val
+        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_${region} --query "[].name" -o tsv); do
+          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=${region} VMSS_NAME=$val
         done
         kubectl get pods -n kube-system -owide
 
diff --git a/.pipelines/cni/lsg/lsg-cni-intergration-template.yaml b/.pipelines/cni/lsg/lsg-cni-intergration-template.yaml
index b6444e4173..0b051d181a 100644
--- a/.pipelines/cni/lsg/lsg-cni-intergration-template.yaml
+++ b/.pipelines/cni/lsg/lsg-cni-intergration-template.yaml
@@ -144,6 +144,7 @@ stages:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               os: ${{ parameters.os }}
               cni: ${{ parameters.cni }}
+              region: $(LOCATION)
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml
index 72830c43c9..e4b1d1983e 100644
--- a/.pipelines/cni/pipeline.yaml
+++ b/.pipelines/cni/pipeline.yaml
@@ -307,6 +307,7 @@ stages:
       clusterName: "cilium-over"
       nodeCount: ${NODE_COUNT_CILIUM}
       vmSize: ${VM_SIZE_CILIUM}
+      arch: amd64
       cni: "cilium"
 
   - template: cilium/cilium-overlay-load-test-template.yaml
@@ -317,6 +318,7 @@ stages:
       hubbleEnabled: true
       nodeCount: ${NODE_COUNT_CILIUM}
       vmSize: ${VM_SIZE_CILIUM}
+      arch: amd64
       cni: "cilium"
 
   - template: cilium/cilium-overlay-load-test-template.yaml
@@ -357,6 +359,7 @@ stages:
       clusterName: "cil-ds-ov"
       nodeCount: ${NODE_COUNT_CILIUM}
       vmSize: ${VM_SIZE_CILIUM}
+      arch: amd64
       dualstackVersion: ${CILIUM_DUALSTACK_VERSION}
       cni: "cilium_dualstack"
 
@@ -449,12 +452,21 @@ stages:
               addSpnToEnvironment: true
               inlineScript: |
                 set -x
+
+                if [[ ${clusterName} =~ 'rdma' ]]; then
+                  region=${LOCATION_RDMA}
+                elif [[ ${clusterName} =~ 'arm' ]]; then
+                  region=${LOCATION_ARM64}
+                else
+                  region=${LOCATION_AMD64}
+                fi
+
                 if [ "$(DELETE_RESOURCES)" ]
                 then
                   echo "Deleting Cluster and resource group"
                   make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=$(clusterName)-$(commitID)
-                  make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
-                  make -C ./hack/aks down AZCLI=az REGION=$(LOCATION) SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=$(clusterName)-$(commitID)
+                  make -C ./hack/aks azcfg AZCLI=az REGION=${region}
+                  make -C ./hack/aks down AZCLI=az REGION=${region} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=$(clusterName)-$(commitID)
                   echo "Cluster and resources down"
                 else
                   echo "Deletion of resources is False"
diff --git a/.pipelines/cni/singletenancy/cniv1-template.yaml b/.pipelines/cni/singletenancy/cniv1-template.yaml
index 96521f942b..78232bee58 100644
--- a/.pipelines/cni/singletenancy/cniv1-template.yaml
+++ b/.pipelines/cni/singletenancy/cniv1-template.yaml
@@ -35,6 +35,10 @@ stages:
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
         npmVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.npmVersion'] ]
+        ${{ if eq(parameters.arch, 'amd64') }}:
+          location: $(LOCATION_AMD64)
+        ${{ if eq(parameters.arch, 'arm64') }}:
+          location: $(LOCATION_ARM64)
       pool:
         name: $(BUILD_POOL_NAME_DEFAULT)
       dependsOn:
@@ -50,7 +54,7 @@ stages:
                 nodeCount: ${{ parameters.nodeCount }}
                 vmSize: ${{ parameters.vmSize }}
                 vmSizeWin: ${{ parameters.vmSizeWin }}
-                region: $(LOCATION)
+                region: $(location)
                 osSKU: ${{ parameters.osSKU }}
 
 # If ensures that only windows template calls are compared against the below condition
@@ -63,6 +67,10 @@ stages:
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
         npmVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.npmVersion'] ]
+        ${{ if eq(parameters.arch, 'amd64') }}:
+          location: $(LOCATION_AMD64)
+        ${{ if eq(parameters.arch, 'arm64') }}:
+          location: $(LOCATION_ARM64)
       pool:
         name: $(BUILD_POOL_NAME_DEFAULT)
       dependsOn:
@@ -79,7 +87,7 @@ stages:
                 nodeCountWin: ${{ parameters.nodeCountWin }}
                 vmSize: ${{ parameters.vmSize }}
                 vmSizeWin: ${{ parameters.vmSizeWin }}
-                region: $(LOCATION)
+                region: $(location)
                 osSKU: ${{ parameters.osSKU }}
                 os: ${{ parameters.os }}
                 osSkuWin: ${{ parameters.osSkuWin }}
@@ -92,6 +100,10 @@ stages:
     variables:
       commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
       npmVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.npmVersion'] ]
+      ${{ if eq(parameters.arch, 'amd64') }}:
+        location: $(LOCATION_AMD64)
+      ${{ if eq(parameters.arch, 'arm64') }}:
+        location: $(LOCATION_ARM64)
     pool:
       name: $(BUILD_POOL_NAME_DEFAULT)
     dependsOn:
@@ -168,6 +180,7 @@ stages:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               os: ${{ parameters.os }}
               cni: cniv1
+              region: $(location)
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -231,6 +244,10 @@ stages:
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
         npmVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.npmVersion'] ]
+        ${{ if eq(parameters.arch, 'amd64') }}:
+          location: $(LOCATION_AMD64)
+        ${{ if eq(parameters.arch, 'arm64') }}:
+          location: $(LOCATION_ARM64)
       pool:
         name: $(BUILD_POOL_NAME_DEFAULT)
       dependsOn:
@@ -275,6 +292,7 @@ stages:
                 os: ${{ parameters.os }}
                 cni: cniv1
                 jobName: restart_nodesHNS
+                region: $(location)
             - template: ../load-test-templates/validate-state-template.yaml
               parameters:
                 clusterName: ${{ parameters.clusterName }}-$(commitID)
diff --git a/.pipelines/cni/singletenancy/cniv2-template.yaml b/.pipelines/cni/singletenancy/cniv2-template.yaml
index 261b44f98c..3e77b829fd 100644
--- a/.pipelines/cni/singletenancy/cniv2-template.yaml
+++ b/.pipelines/cni/singletenancy/cniv2-template.yaml
@@ -33,6 +33,12 @@ stages:
       condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cniv2'), contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'linux'), contains(variables.CONTROL_OS, 'all') ) ) )
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
+        ${{ if contains(parameters.clusterName, 'rdma') }}:
+          location: $(LOCATION_RDMA)
+        ${{ elseif eq(parameters.arch, 'arm64') }}:
+          location: $(LOCATION_ARM64)
+        ${{ else }}:
+          location: $(LOCATION_AMD64)
       pool:
         name: $(BUILD_POOL_NAME_DEFAULT)
       dependsOn:
@@ -47,7 +53,7 @@ stages:
                 clusterName: ${{ parameters.clusterName }}-$(commitID)
                 nodeCount: ${{ parameters.nodeCount }}
                 vmSize: ${{ parameters.vmSize }}
-                region: $(LOCATION)
+                region: $(location)
                 osSKU: ${{ parameters.osSKU }}
 
 # If ensures that only windows template calls are compared against the below condition
@@ -59,6 +65,12 @@ stages:
       condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cniv2'), contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'windows'), contains(variables.CONTROL_OS, 'all') ) ) )
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
+        ${{ if contains(parameters.clusterName, 'rdma') }}:
+          location: $(LOCATION_RDMA)
+        ${{ elseif eq(parameters.arch, 'arm64') }}:
+          location: $(LOCATION_ARM64)
+        ${{ else }}:
+          location: $(LOCATION_AMD64)
       pool:
         name: $(BUILD_POOL_NAME_DEFAULT)
       dependsOn:
@@ -75,7 +87,7 @@ stages:
                 nodeCountWin: ${{ parameters.nodeCountWin }}
                 vmSize: ${{ parameters.vmSize }}
                 vmSizeWin: ${{ parameters.vmSizeWin }}
-                region: $(LOCATION)
+                region: $(location)
                 osSKU: ${{ parameters.osSKU }}
                 os: ${{ parameters.os }}
                 osSkuWin: ${{ parameters.osSkuWin }}
@@ -93,6 +105,12 @@ stages:
         nodeCount: ${{ parameters.nodeCountWin }}
       ${{ else }}:
         nodeCount: ${{ parameters.nodeCount }}
+      ${{ if contains(parameters.clusterName, 'rdma') }}:
+        location: $(LOCATION_RDMA)
+      ${{ elseif eq(parameters.arch, 'arm64') }}:
+        location: $(LOCATION_ARM64)
+      ${{ else }}:
+        location: $(LOCATION_AMD64)
     pool:
       name: $(BUILD_POOL_NAME_DEFAULT)
     dependsOn:
@@ -204,6 +222,7 @@ stages:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
               os: ${{ parameters.os }}
               cni: cniv2
+              region: $(location)
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -277,6 +296,12 @@ stages:
     - stage: ${{ parameters.name }}_HNS
       variables:
         commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
+        ${{ if contains(parameters.clusterName, 'rdma') }}:
+          location: $(LOCATION_RDMA)
+        ${{ elseif eq(parameters.arch, 'arm64') }}:
+          location: $(LOCATION_ARM64)
+        ${{ else }}:
+          location: $(LOCATION_AMD64)
       pool:
         name: $(BUILD_POOL_NAME_DEFAULT)
       dependsOn:
@@ -320,6 +345,7 @@ stages:
                 nodeCount: ${{ parameters.nodeCountWin }}
                 scaleup: ${{ parameters.scaleup }}
                 jobName: "HNS_restart_nodes"
+                region: $(location)
             - template: ../load-test-templates/validate-state-template.yaml
               parameters:
                 clusterName: ${{ parameters.clusterName }}-$(commitID)