From ff1fbf503b37196c604e8f444c619d62c6aa577a Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 12:09:03 +0100 Subject: [PATCH 01/71] Changed DNS resources to data resources --- terraform/dns.tf | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/terraform/dns.tf b/terraform/dns.tf index 5b7b801..a5ebed0 100644 --- a/terraform/dns.tf +++ b/terraform/dns.tf @@ -1,17 +1,11 @@ # DNS -resource "azurerm_resource_group" "dns" { +data "azurerm_resource_group" "dns" { name = var.dns_resource_group_name - location = var.location - tags = var.tags - - lifecycle { - ignore_changes = [tags] - } } -resource "azurerm_dns_zone" "dns" { +data "azurerm_dns_zone" "dns" { name = var.dns_zone_name - resource_group_name = azurerm_resource_group.dns.name + resource_group_name = data.azurerm_resource_group.dns.name } @@ -48,7 +42,7 @@ resource "azuread_service_principal_password" "aks_dns_sp" { resource "azurerm_role_assignment" "aks_dns_sp_to_rg" { principal_id = azuread_service_principal.aks_dns_sp.id role_definition_name = "Reader" - scope = azurerm_dns_zone.dns.id + scope = data.azurerm_dns_zone.dns.id skip_service_principal_aad_check = true depends_on = [azuread_service_principal_password.aks_dns_sp] } @@ -57,7 +51,7 @@ resource "azurerm_role_assignment" "aks_dns_sp_to_rg" { resource "azurerm_role_assignment" "aks_dns_sp_to_zone" { principal_id = azuread_service_principal.aks_dns_sp.id role_definition_name = "Contributor" - scope = azurerm_resource_group.dns.id + scope = data.azurerm_resource_group.dns.id skip_service_principal_aad_check = true depends_on = [azuread_service_principal_password.aks_dns_sp] } @@ -77,7 +71,7 @@ resource "kubernetes_secret" "external_dns" { "aadClientSecret": "${random_string.aks_dns_sp.result}", "tenantId": "${data.azurerm_subscription.current.tenant_id}", "subscriptionId": "${data.azurerm_subscription.current.subscription_id}", - "resourceGroup": "${azurerm_resource_group.dns.name}" + "resourceGroup": "${data.azurerm_resource_group.dns.name}" } EOT } From d6f0d242886e54f56273c52b639e4c2c97992ab0 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 12:30:07 +0100 Subject: [PATCH 02/71] Changed AKS to enable RBAC --- terraform/aks.tf | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/terraform/aks.tf b/terraform/aks.tf index 2bc5f21..74d00f5 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -98,16 +98,16 @@ resource "azurerm_kubernetes_cluster" "aks" { } # TODO Enable RBAC and AAD auth: https://app.zenhub.com/workspaces/aks-nexus-velero-5e602702ee332f0fc76d35dd/issues/adamrushuk/aks-nexus-velero/105 - # role_based_access_control { - # enabled = true - - # azure_active_directory { - # managed = true - # admin_group_object_ids = [ - # data.azuread_group.aks.id - # ] - # } - # } + role_based_access_control { + enabled = true + + # azure_active_directory { + # managed = true + # admin_group_object_ids = [ + # data.azuread_group.aks.id + # ] + # } + } addon_profile { kube_dashboard { From 0cee576a96032e57438661779e6fb84a13b97142 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 12:40:04 +0100 Subject: [PATCH 03/71] Changed DNS_RG_NAME to be hard-coded env var --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aec9907..4a5e8bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,6 +39,7 @@ env: DEMO_USER_USERNAME: demo_user # DEMO_USER_PASSWORD: ${{ secrets.DEMO_USER_PASSWORD }} DNS_DOMAIN_NAME: nexus.thehypepipe.co.uk + DNS_RG_NAME: rg-dns DOCKER_FQDN: docker.thehypepipe.co.uk EMAIL_ADDRESS: certadmin@domain.com ENABLE_TLS_INGRESS: true @@ -87,7 +88,6 @@ jobs: chmod -R +x ./scripts/ echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 - echo ::set-env name=DNS_RG_NAME::${{ env.PREFIX }}-rg-externaldns echo ::set-env name=TERRAFORM_STORAGE_ACCOUNT::${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001 echo ::set-env name=TERRAFORM_STORAGE_RG::${{ env.PREFIX }}-rg-tfstate-dev-001 echo ::set-env name=VELERO_STORAGE_RG::${{ env.PREFIX }}-rg-velero-dev-001 From cf5587c37ff2d809de68c14c83f772eaf44b17e2 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 15:34:02 +0100 Subject: [PATCH 04/71] Removed k8s dashboard --- terraform/aks.tf | 4 ---- terraform/variables.tf | 11 ++++++----- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/terraform/aks.tf b/terraform/aks.tf index 74d00f5..3632deb 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -110,10 +110,6 @@ resource "azurerm_kubernetes_cluster" "aks" { } addon_profile { - kube_dashboard { - enabled = var.aks_dashboard_enabled - } - oms_agent { enabled = var.aks_container_insights_enabled log_analytics_workspace_id = var.aks_container_insights_enabled ? azurerm_log_analytics_workspace.aks[0].id : null diff --git a/terraform/variables.tf b/terraform/variables.tf index cd28e85..57ccf9a 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -34,6 +34,12 @@ variable "velero_chart_version" { variable "nexus_chart_version" { default = "0.2.6" } + +# https://github.com/SparebankenVest/public-helm-charts/releases +# https://github.com/SparebankenVest/helm-charts/tree/gh-pages/akv2k8s +variable "akv2k8s_chart_version" { + default = "1.1.25" +} #endregion Versions @@ -95,11 +101,6 @@ variable "sla_sku" { default = "Free" } -variable "aks_dashboard_enabled" { - description = "Should Kubernetes dashboard be enabled" - default = false -} - variable "aks_container_insights_enabled" { description = "Should Container Insights monitoring be enabled" default = false From b11025d7beac01f6c97cc847dd2bd468fc621bc8 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 15:34:42 +0100 Subject: [PATCH 05/71] Added depends_on for AKS cluster --- terraform/helm_nexus.tf | 2 ++ terraform/helm_nginx.tf | 2 ++ terraform/velero.tf | 2 ++ 3 files changed, 6 insertions(+) diff --git a/terraform/helm_nexus.tf b/terraform/helm_nexus.tf index 5b6a090..ce22a80 100644 --- a/terraform/helm_nexus.tf +++ b/terraform/helm_nexus.tf @@ -8,6 +8,8 @@ resource "kubernetes_namespace" "nexus" { timeouts { delete = "15m" } + + depends_on = [azurerm_kubernetes_cluster.aks] } # https://www.terraform.io/docs/providers/helm/r/release.html diff --git a/terraform/helm_nginx.tf b/terraform/helm_nginx.tf index d858e73..fe64308 100644 --- a/terraform/helm_nginx.tf +++ b/terraform/helm_nginx.tf @@ -8,6 +8,8 @@ resource "kubernetes_namespace" "ingress" { timeouts { delete = "15m" } + + depends_on = [azurerm_kubernetes_cluster.aks] } # https://www.terraform.io/docs/providers/helm/r/release.html diff --git a/terraform/velero.tf b/terraform/velero.tf index 5e792df..f20b8bf 100644 --- a/terraform/velero.tf +++ b/terraform/velero.tf @@ -51,6 +51,8 @@ resource "kubernetes_namespace" "velero" { timeouts { delete = "15m" } + + depends_on = [azurerm_kubernetes_cluster.aks] } resource "kubernetes_secret" "velero_credentials" { From d062f158b14b54da3d5c127dad9e2ddd6e019583 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 15:35:04 +0100 Subject: [PATCH 06/71] Added akv2k8s helm chart --- terraform/helm_akv2k8s.tf | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 terraform/helm_akv2k8s.tf diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf new file mode 100644 index 0000000..464b0ac --- /dev/null +++ b/terraform/helm_akv2k8s.tf @@ -0,0 +1,43 @@ +# Azure Key Vault to Kubernetes (akv2k8s) makes Azure Key Vault secrets, certificates and keys available in +# Kubernetes and/or your application - in a simple and secure way +# +# https://akv2k8s.io/ +# https://github.com/SparebankenVest/azure-key-vault-to-kubernetes + + +# https://www.terraform.io/docs/provisioners/local-exec.html +resource "null_resource" "akv2k8s_crds" { + # triggers = { + # always_run = "${timestamp()}" + # } + + provisioner "local-exec" { + command = "kubectl apply -f https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml" + interpreter = ["/bin/bash", "-c"] + } + depends_on = [azurerm_kubernetes_cluster.aks] +} + +# https://www.terraform.io/docs/providers/kubernetes/r/namespace.html +resource "kubernetes_namespace" "akv2k8s" { + metadata { + name = "akv2k8s" + } + timeouts { + delete = "15m" + } + + depends_on = [null_resource.akv2k8s_crds] +} + +# https://www.terraform.io/docs/providers/helm/r/release.html +# https://github.com/SparebankenVest/public-helm-charts/tree/master/stable/akv2k8s#configuration +resource "helm_release" "akv2k8s" { + chart = "akv2k8s" + name = "akv2k8s" + namespace = "akv2k8s" + repository = "http://charts.spvapi.no" + version = var.akv2k8s_chart_version + timeout = 600 + depends_on = [kubernetes_namespace.akv2k8s] +} From ac199ce509421d0488dc575b01194c30d6068930 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 16:02:47 +0100 Subject: [PATCH 07/71] Debugging akv2k8s kubectl --- terraform/files/AzureKeyVaultSecret.yaml | 101 +++++++++++++++++++++++ terraform/helm_akv2k8s.tf | 12 +-- terraform/variables.tf | 7 ++ 3 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 terraform/files/AzureKeyVaultSecret.yaml diff --git a/terraform/files/AzureKeyVaultSecret.yaml b/terraform/files/AzureKeyVaultSecret.yaml new file mode 100644 index 0000000..fe562da --- /dev/null +++ b/terraform/files/AzureKeyVaultSecret.yaml @@ -0,0 +1,101 @@ +# https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: azurekeyvaultsecrets.spv.no + labels: + app.kubernetes.io/name: akv2k8s + annotations: + "helm.sh/resource-policy": keep +spec: + group: spv.no + names: + kind: AzureKeyVaultSecret + listKind: AzureKeyVaultSecretList + plural: azurekeyvaultsecrets + singular: azurekeyvaultsecret + shortNames: + - akvs + categories: + - all + additionalPrinterColumns: + - name: Vault + type: string + description: Which Azure Key Vault this resource is asosiated with + JSONPath: .spec.vault.name + - name: Vault Object + type: string + description: Which Azure Key Vault object this resource is asosiated with + JSONPath: .spec.vault.object.name + - name: Secret Name + type: string + description: Which Kubernetes Secret this resource is synched with, if any + JSONPath: .status.secretName + - name: Synched + type: string + description: When this resource was last synched with Azure Key Vault + JSONPath: .status.lastAzureUpdate + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: false + - name: v1 + served: true + storage: true + + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + required: ['vault'] + properties: + vault: + required: ['name', 'object'] + properties: + name: + type: string + description: Name of the Azure Key Vault + object: + required: ['name', 'type'] + properties: + name: + type: string + description: The object name in Azure Key Vault + type: + type: string + description: The type of object in Azure Key Vault + enum: + - secret + - certificate + - key + - multi-key-value-secret + version: + type: string + description: The object version in Azure Key Vault + contentType: + type: string + description: Only used when type is multi-key-value-secret. + enum: + - application/x-json + - application/x-yaml + output: + properties: + transform: + type: array + items: + type: string + secret: + required: ['name'] + properties: + name: + type: string + description: Name for Kubernetes secret + type: + type: string + description: Type of Secret in Kubernetes + dataKey: + type: string + description: The key to use in Kubernetes secret when setting the value from Azure Keyv Vault object data diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 464b0ac..9c45933 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -7,14 +7,16 @@ # https://www.terraform.io/docs/provisioners/local-exec.html resource "null_resource" "akv2k8s_crds" { - # triggers = { - # always_run = "${timestamp()}" - # } + triggers = { + # always_run = "${timestamp()}" + akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) + } provisioner "local-exec" { - command = "kubectl apply -f https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml" - interpreter = ["/bin/bash", "-c"] + # command = "kubectl apply -f https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml" + command = "kubectl cluster-info" } + depends_on = [azurerm_kubernetes_cluster.aks] } diff --git a/terraform/variables.tf b/terraform/variables.tf index 57ccf9a..65aacdd 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -268,3 +268,10 @@ variable "nexus_ingress_enabled" { variable "nexus_letsencrypt_environment" { default = "__CERT_API_ENVIRONMENT__" } + + + +# akv2k8s +variable "akv2k8s_yaml_path" { + default = "files/AzureKeyVaultSecret.yaml" +} From 22740706f2e1a47eba884eec7ac2d657ba7f6f67 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 16:35:14 +0100 Subject: [PATCH 08/71] Debugging kubectl --- terraform/helm_akv2k8s.tf | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 9c45933..dd09609 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -8,13 +8,20 @@ # https://www.terraform.io/docs/provisioners/local-exec.html resource "null_resource" "akv2k8s_crds" { triggers = { - # always_run = "${timestamp()}" - akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) + always_run = "${timestamp()}" + # akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) } provisioner "local-exec" { # command = "kubectl apply -f https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml" command = "kubectl cluster-info" + + command = < ./azurek8s + export KUBECONFIG=./azurek8s + kubectl cluster-info + EOT + } depends_on = [azurerm_kubernetes_cluster.aks] From 65ec3cd5dbe79889413d9efbb449be4ecb32217f Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 16:39:56 +0100 Subject: [PATCH 09/71] debug --- terraform/helm_akv2k8s.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index dd09609..fa5999f 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -14,7 +14,7 @@ resource "null_resource" "akv2k8s_crds" { provisioner "local-exec" { # command = "kubectl apply -f https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml" - command = "kubectl cluster-info" + # command = "kubectl cluster-info" command = < ./azurek8s From 1b0b687f85e9a89e05e408477dd9cd8756f7b8bb Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 17:06:18 +0100 Subject: [PATCH 10/71] Fixed null_resource --- terraform/helm_akv2k8s.tf | 21 ++++++++++++--------- terraform/variables.tf | 4 ++++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index fa5999f..60bebc8 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -4,27 +4,30 @@ # https://akv2k8s.io/ # https://github.com/SparebankenVest/azure-key-vault-to-kubernetes +resource "local_file" "kubeconfig" { + sensitive_content = azurerm_kubernetes_cluster.aks.kube_config_raw + filename = var.aks_config_path + + depends_on = [azurerm_kubernetes_cluster.aks] +} # https://www.terraform.io/docs/provisioners/local-exec.html resource "null_resource" "akv2k8s_crds" { triggers = { - always_run = "${timestamp()}" - # akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) + # always_run = "${timestamp()}" + kubeconfig_contents = filemd5(var.aks_config_path) + akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) } provisioner "local-exec" { - # command = "kubectl apply -f https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml" - # command = "kubectl cluster-info" - command = < ./azurek8s - export KUBECONFIG=./azurek8s + export KUBECONFIG=${var.aks_config_path} kubectl cluster-info + kubectl apply -f ${file(var.akv2k8s_yaml_path)} EOT - } - depends_on = [azurerm_kubernetes_cluster.aks] + depends_on = [local_file.kubeconfig] } # https://www.terraform.io/docs/providers/kubernetes/r/namespace.html diff --git a/terraform/variables.tf b/terraform/variables.tf index 65aacdd..2c58a09 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -106,6 +106,10 @@ variable "aks_container_insights_enabled" { default = false } +variable "aks_config_path" { + default = "${path.module}/files/azurek8s_config" +} + # TODO DELETE SECTION # Service Principle for AKS From 0327a4871ffaf87f18e3a037763da650691a4175 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 17:11:34 +0100 Subject: [PATCH 11/71] Fixed var error for aks_config_path --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 2c58a09..fea4bac 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -107,7 +107,7 @@ variable "aks_container_insights_enabled" { } variable "aks_config_path" { - default = "${path.module}/files/azurek8s_config" + default = "./azurek8s_config" } From 2365ea346b70e001b52269e3a42204994cd7c93a Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 17:16:43 +0100 Subject: [PATCH 12/71] debug --- terraform/helm_akv2k8s.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 60bebc8..96eef39 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -14,8 +14,7 @@ resource "local_file" "kubeconfig" { # https://www.terraform.io/docs/provisioners/local-exec.html resource "null_resource" "akv2k8s_crds" { triggers = { - # always_run = "${timestamp()}" - kubeconfig_contents = filemd5(var.aks_config_path) + always_run = "${timestamp()}" akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) } From e94cde96537d4a681e61445885beb96584d8341d Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 17:24:10 +0100 Subject: [PATCH 13/71] Changed interpreter to use bash --- terraform/helm_akv2k8s.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 96eef39..50437eb 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -19,6 +19,7 @@ resource "null_resource" "akv2k8s_crds" { } provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] command = < Date: Sun, 18 Oct 2020 17:32:45 +0100 Subject: [PATCH 14/71] Fixed akv2k8s_crds --- terraform/helm_akv2k8s.tf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 50437eb..8ede591 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -14,7 +14,7 @@ resource "local_file" "kubeconfig" { # https://www.terraform.io/docs/provisioners/local-exec.html resource "null_resource" "akv2k8s_crds" { triggers = { - always_run = "${timestamp()}" + # always_run = "${timestamp()}" akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) } @@ -22,8 +22,7 @@ resource "null_resource" "akv2k8s_crds" { interpreter = ["/bin/bash", "-c"] command = < Date: Sun, 18 Oct 2020 18:08:03 +0100 Subject: [PATCH 15/71] Added cert_sync_yaml --- terraform/files/akvs-certificate-sync.yaml | 16 ++++++++++++++++ terraform/helm_akv2k8s.tf | 2 ++ terraform/variables.tf | 4 ++++ 3 files changed, 22 insertions(+) create mode 100644 terraform/files/akvs-certificate-sync.yaml diff --git a/terraform/files/akvs-certificate-sync.yaml b/terraform/files/akvs-certificate-sync.yaml new file mode 100644 index 0000000..c57de77 --- /dev/null +++ b/terraform/files/akvs-certificate-sync.yaml @@ -0,0 +1,16 @@ +# https://akv2k8s.io/tutorials/sync/2-certificate/ +apiVersion: spv.no/v1alpha1 +kind: AzureKeyVaultSecret +metadata: + name: certificate-sync + namespace: nexus +spec: + vault: + name: kv-rush-iz6y # name of key vault + object: + name: wildcard-thehypepipe-co-uk # key vault certificate name + type: certificate + output: + secret: + name: my-certificate-from-akv # kubernetes secret name + type: kubernetes.io/tls # kubernetes secret type diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 8ede591..edfb1d0 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -16,6 +16,7 @@ resource "null_resource" "akv2k8s_crds" { triggers = { # always_run = "${timestamp()}" akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) + cert_sync_yaml_contents = filemd5(var.cert_sync_yaml_path) } provisioner "local-exec" { @@ -23,6 +24,7 @@ resource "null_resource" "akv2k8s_crds" { command = < Date: Sun, 18 Oct 2020 18:50:34 +0100 Subject: [PATCH 16/71] Added DNS_RG_NAME --- .github/workflows/destroy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 027cdee..d02fac6 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -33,6 +33,7 @@ env: # prod or staging CERT_API_ENVIRONMENT: staging DNS_DOMAIN_NAME: nexus.thehypepipe.co.uk + DNS_RG_NAME: rg-dns EMAIL_ADDRESS: certadmin@domain.com ENABLE_TLS_INGRESS: true FORCE_TEST_FAIL: false From 1e8bbf549cc576a86936072f7c048451f0dc053e Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 18:56:37 +0100 Subject: [PATCH 17/71] Fixed destroy workflow --- .github/workflows/destroy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index d02fac6..9f692d0 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -39,8 +39,9 @@ env: FORCE_TEST_FAIL: false HAS_SUBDOMAIN: true LOCATION: uksouth + ROOT_DOMAIN_NAME: thehypepipe.co.uk # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: false + VELERO_ENABLED: true # terraform TF_IN_AUTOMATION: "true" From 4d28ad423dfb626cdefc5c934526fb4584b695d5 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 19:08:25 +0100 Subject: [PATCH 18/71] Changed nexus_chart_version to 0.2.7 --- .github/workflows/build.yml | 4 ++-- terraform/helm_nexus.tf | 5 +++++ terraform/variables.tf | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a5e8bf..39acbf4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,8 +34,8 @@ env: ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }} # other - # prod or staging - CERT_API_ENVIRONMENT: prod + # prod or staging. "" disables cert-manager annotations + CERT_API_ENVIRONMENT: "" DEMO_USER_USERNAME: demo_user # DEMO_USER_PASSWORD: ${{ secrets.DEMO_USER_PASSWORD }} DNS_DOMAIN_NAME: nexus.thehypepipe.co.uk diff --git a/terraform/helm_nexus.tf b/terraform/helm_nexus.tf index ce22a80..cdc13e0 100644 --- a/terraform/helm_nexus.tf +++ b/terraform/helm_nexus.tf @@ -41,6 +41,11 @@ resource "helm_release" "nexus" { value = var.nexus_letsencrypt_environment } + set { + name = "ingress.tls.secretName" + value = "nexus-tls-secret" + } + timeout = 600 depends_on = [helm_release.nginx, kubernetes_namespace.nexus] } diff --git a/terraform/variables.tf b/terraform/variables.tf index 2460749..df1fa88 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -32,7 +32,7 @@ variable "velero_chart_version" { # https://github.com/adamrushuk/charts/releases variable "nexus_chart_version" { - default = "0.2.6" + default = "0.2.7" } # https://github.com/SparebankenVest/public-helm-charts/releases From 00af492cbb8e9bb3c5bd1ad1d4c8457d86861bb9 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 20:19:51 +0100 Subject: [PATCH 19/71] Added key vault vars and access policy --- .github/workflows/build.yml | 2 + .github/workflows/destroy.yml | 2 + terraform/aks.tf | 43 +++++++++++++++++----- terraform/files/akvs-certificate-sync.yaml | 2 +- terraform/variables.tf | 8 ++++ 5 files changed, 46 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39acbf4..7c8a6a4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,8 @@ env: ENABLE_TLS_INGRESS: true FORCE_TEST_FAIL: false HAS_SUBDOMAIN: true + KEY_VAULT_NAME: kv-rush-iz6y + KEY_VAULT_RESOURCE_GROUP_NAME: rg-keyvault-acmebot LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk # NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }} diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 9f692d0..f346a91 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -38,6 +38,8 @@ env: ENABLE_TLS_INGRESS: true FORCE_TEST_FAIL: false HAS_SUBDOMAIN: true + KEY_VAULT_NAME: kv-rush-iz6y + KEY_VAULT_RESOURCE_GROUP_NAME: rg-keyvault-acmebot LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' diff --git a/terraform/aks.tf b/terraform/aks.tf index 3632deb..f9c8c7d 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -82,16 +82,6 @@ resource "azurerm_kubernetes_cluster" "aks" { } } - - # TODO DELETE SECTION - # service_principal block: https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html#client_id - # service_principal { - # client_id = var.service_principal_client_id - # client_secret = var.service_principal_client_secret - # } - # TODO DELETE SECTION - - # managed identity block: https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html#type-1 identity { type = "SystemAssigned" @@ -127,3 +117,36 @@ resource "azurerm_kubernetes_cluster" "aks" { ] } } + + +# Key vault access policy for AKS +data "azurerm_key_vault" "kv" { + name = var.key_vault_name + resource_group_name = var.key_vault_resource_group_name +} + +resource "azurerm_key_vault_access_policy" "aks" { + key_vault_id = data.azurerm_key_vault.kv.id + + tenant_id = data.azurerm_subscription.current.tenant_id + object_id = azurerm_kubernetes_cluster.aks.identity[0].principal_id + + certificate_permissions = [ + "backup", + "create", + "delete", + "deleteissuers", + "get", + "getissuers", + "import", + "list", + "listissuers", + "managecontacts", + "manageissuers", + "purge", + "recover", + "restore", + "setissuers", + "update" + ] +} diff --git a/terraform/files/akvs-certificate-sync.yaml b/terraform/files/akvs-certificate-sync.yaml index c57de77..e8b25fa 100644 --- a/terraform/files/akvs-certificate-sync.yaml +++ b/terraform/files/akvs-certificate-sync.yaml @@ -12,5 +12,5 @@ spec: type: certificate output: secret: - name: my-certificate-from-akv # kubernetes secret name + name: nexus-tls-secret # kubernetes secret name type: kubernetes.io/tls # kubernetes secret type diff --git a/terraform/variables.tf b/terraform/variables.tf index df1fa88..a2ccad4 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -82,6 +82,14 @@ variable "tags" { } } +variable "key_vault_name" { + default = "__KEY_VAULT_NAME__" +} + +variable "key_vault_resource_group_name" { + default = "__KEY_VAULT_RESOURCE_GROUP_NAME__" +} + # AKS From 57c2d6398fff5bce81a7604a5ccf94d6811349cd Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 20:37:38 +0100 Subject: [PATCH 20/71] Debugging --- terraform/outputs.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 2e8b498..a3411fe 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -1,7 +1,7 @@ # WARNING: this outputs credential / login config -# output "aks_config" { -# value = azurerm_kubernetes_cluster.aks -# } +output "aks_config" { + value = azurerm_kubernetes_cluster.aks +} output "aks_credentials_command" { value = "az aks get-credentials --resource-group ${azurerm_kubernetes_cluster.aks.resource_group_name} --name ${azurerm_kubernetes_cluster.aks.name} --overwrite-existing" From e8526d2090a38496bde67f37ad9275ce45fe2ac4 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 20:48:01 +0100 Subject: [PATCH 21/71] Changed to use aks kubelet_identity --- terraform/aks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/aks.tf b/terraform/aks.tf index f9c8c7d..17a1908 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -129,7 +129,7 @@ resource "azurerm_key_vault_access_policy" "aks" { key_vault_id = data.azurerm_key_vault.kv.id tenant_id = data.azurerm_subscription.current.tenant_id - object_id = azurerm_kubernetes_cluster.aks.identity[0].principal_id + object_id = azurerm_kubernetes_cluster.aks.kubelet_identity.object_id certificate_permissions = [ "backup", From 3405eaa10074ea7082cf56d747babf3b0b120556 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 20:51:48 +0100 Subject: [PATCH 22/71] Added kube_dashboard fix --- terraform/aks.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/terraform/aks.tf b/terraform/aks.tf index 17a1908..39a9d94 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -100,6 +100,12 @@ resource "azurerm_kubernetes_cluster" "aks" { } addon_profile { + # cannot remove this deprecated block yet, due to this issue: + # https://github.com/terraform-providers/terraform-provider-azurerm/issues/7716 + kube_dashboard { + enabled = false + } + oms_agent { enabled = var.aks_container_insights_enabled log_analytics_workspace_id = var.aks_container_insights_enabled ? azurerm_log_analytics_workspace.aks[0].id : null From ac960aa73da32309811af32f657fe0c006dc5575 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 18 Oct 2020 20:56:53 +0100 Subject: [PATCH 23/71] Fixed kubelet_identity id --- terraform/aks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/aks.tf b/terraform/aks.tf index 39a9d94..72d4ba4 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -135,7 +135,7 @@ resource "azurerm_key_vault_access_policy" "aks" { key_vault_id = data.azurerm_key_vault.kv.id tenant_id = data.azurerm_subscription.current.tenant_id - object_id = azurerm_kubernetes_cluster.aks.kubelet_identity.object_id + object_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id certificate_permissions = [ "backup", From 91fb45dd8135b937b5c6a900785a8e8b43b3fe89 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Mon, 19 Oct 2020 07:39:01 +0100 Subject: [PATCH 24/71] Bumped app and provider versions --- terraform/providers.tf | 4 ++-- terraform/variables.tf | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/terraform/providers.tf b/terraform/providers.tf index caa1cb3..376a756 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -11,7 +11,7 @@ terraform { # versioning syntax: https://www.terraform.io/docs/configuration/modules.html#module-versions required_providers { # https://github.com/hashicorp/terraform-provider-helm/releases - helm = "1.3.1" + helm = "1.3.2" # https://github.com/hashicorp/terraform-provider-kubernetes/releases kubernetes = "1.13.2" @@ -30,7 +30,7 @@ terraform { # must include blank features block # https://github.com/terraform-providers/terraform-provider-azurerm/releases provider "azurerm" { - version = "2.30.0" + version = "2.32.0" features {} } diff --git a/terraform/variables.tf b/terraform/variables.tf index a2ccad4..951f5f8 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -17,17 +17,17 @@ variable "kubernetes_version" { # https://github.com/kubernetes/ingress-nginx/releases # https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/Chart.yaml#L3 variable "nginx_chart_version" { - default = "3.4.0" + default = "3.7.1" } # https://hub.helm.sh/charts/jetstack/cert-manager variable "cert_manager_chart_version" { - default = "v1.0.2" + default = "v1.0.3" } # https://github.com/vmware-tanzu/helm-charts/releases variable "velero_chart_version" { - default = "2.12.17" + default = "2.13.3" } # https://github.com/adamrushuk/charts/releases @@ -37,6 +37,7 @@ variable "nexus_chart_version" { # https://github.com/SparebankenVest/public-helm-charts/releases # https://github.com/SparebankenVest/helm-charts/tree/gh-pages/akv2k8s +# https://github.com/SparebankenVest/public-helm-charts/blob/master/stable/akv2k8s/Chart.yaml#L5 variable "akv2k8s_chart_version" { default = "1.1.25" } From 4ee964be1f8759ec7f21def53168daa178d66f80 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Mon, 19 Oct 2020 08:10:13 +0100 Subject: [PATCH 25/71] Added full get perms for akv2k8s --- terraform/aks.tf | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/terraform/aks.tf b/terraform/aks.tf index 72d4ba4..b492431 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -125,7 +125,7 @@ resource "azurerm_kubernetes_cluster" "aks" { } -# Key vault access policy for AKS +# Key vault access policy for AKS / akv2k8s data "azurerm_key_vault" "kv" { name = var.key_vault_name resource_group_name = var.key_vault_resource_group_name @@ -138,21 +138,14 @@ resource "azurerm_key_vault_access_policy" "aks" { object_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id certificate_permissions = [ - "backup", - "create", - "delete", - "deleteissuers", - "get", - "getissuers", - "import", - "list", - "listissuers", - "managecontacts", - "manageissuers", - "purge", - "recover", - "restore", - "setissuers", - "update" + "get" + ] + + key_permissions = [ + "get" + ] + + secret_permissions = [ + "get" ] } From 9d8a6f5378793275da515811eee4a04beff85e85 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Mon, 19 Oct 2020 08:46:59 +0100 Subject: [PATCH 26/71] Added token replacement for k8s tls secret --- .github/workflows/build.yml | 2 ++ .github/workflows/destroy.yml | 2 ++ scripts/Replace-Tokens.ps1 | 2 +- terraform/files/akvs-certificate-sync.yaml | 6 +++--- terraform/helm_nexus.tf | 2 +- terraform/variables.tf | 4 ++++ 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c8a6a4..b2429f0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,9 @@ env: ENABLE_TLS_INGRESS: true FORCE_TEST_FAIL: false HAS_SUBDOMAIN: true + K8S_TLS_SECRET_NAME: tls-secret KEY_VAULT_NAME: kv-rush-iz6y + KEY_VAULT_CERT_NAME: wildcard-thehypepipe-co-uk KEY_VAULT_RESOURCE_GROUP_NAME: rg-keyvault-acmebot LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index f346a91..d0b5d89 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -38,7 +38,9 @@ env: ENABLE_TLS_INGRESS: true FORCE_TEST_FAIL: false HAS_SUBDOMAIN: true + K8S_TLS_SECRET_NAME: tls-secret KEY_VAULT_NAME: kv-rush-iz6y + KEY_VAULT_CERT_NAME: wildcard-thehypepipe-co-uk KEY_VAULT_RESOURCE_GROUP_NAME: rg-keyvault-acmebot LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk diff --git a/scripts/Replace-Tokens.ps1 b/scripts/Replace-Tokens.ps1 index 11db016..11fb741 100755 --- a/scripts/Replace-Tokens.ps1 +++ b/scripts/Replace-Tokens.ps1 @@ -1,6 +1,6 @@ # Replace tokens param ( - $TargetFilePattern = './terraform/*.tf', + $TargetFilePattern = @('./terraform/*.tf', './terraform/files/*.yaml'), $TokenPrefix = '__', $TokenSuffix = '__', diff --git a/terraform/files/akvs-certificate-sync.yaml b/terraform/files/akvs-certificate-sync.yaml index e8b25fa..9bd60f8 100644 --- a/terraform/files/akvs-certificate-sync.yaml +++ b/terraform/files/akvs-certificate-sync.yaml @@ -6,11 +6,11 @@ metadata: namespace: nexus spec: vault: - name: kv-rush-iz6y # name of key vault + name: __KEY_VAULT_NAME__ # name of key vault object: - name: wildcard-thehypepipe-co-uk # key vault certificate name + name: __KEY_VAULT_CERT_NAME__ # key vault certificate name type: certificate output: secret: - name: nexus-tls-secret # kubernetes secret name + name: __K8S_TLS_SECRET_NAME__ # kubernetes secret name type: kubernetes.io/tls # kubernetes secret type diff --git a/terraform/helm_nexus.tf b/terraform/helm_nexus.tf index cdc13e0..cd0f357 100644 --- a/terraform/helm_nexus.tf +++ b/terraform/helm_nexus.tf @@ -43,7 +43,7 @@ resource "helm_release" "nexus" { set { name = "ingress.tls.secretName" - value = "nexus-tls-secret" + value = var.nexus_tls_secret_name } timeout = 600 diff --git a/terraform/variables.tf b/terraform/variables.tf index 951f5f8..ea04343 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -282,6 +282,10 @@ variable "nexus_letsencrypt_environment" { default = "__CERT_API_ENVIRONMENT__" } +variable "nexus_tls_secret_name" { + default = "__K8S_TLS_SECRET_NAME__" +} + # akv2k8s From 06b816263646a79a8912d2ccc45f7a44d8c74348 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Mon, 19 Oct 2020 09:34:35 +0100 Subject: [PATCH 27/71] Changed nginx back to 3.4.0 --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index ea04343..33d3a4a 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -17,7 +17,7 @@ variable "kubernetes_version" { # https://github.com/kubernetes/ingress-nginx/releases # https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/Chart.yaml#L3 variable "nginx_chart_version" { - default = "3.7.1" + default = "3.4.0" } # https://hub.helm.sh/charts/jetstack/cert-manager From 7a73af7b9f0b0984c29009802952e0ddb84d201a Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 20 Oct 2020 07:47:39 +0100 Subject: [PATCH 28/71] Added chainOrder for AzureKeyVaultSecret sync --- terraform/files/akvs-certificate-sync.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/files/akvs-certificate-sync.yaml b/terraform/files/akvs-certificate-sync.yaml index 9bd60f8..33b5682 100644 --- a/terraform/files/akvs-certificate-sync.yaml +++ b/terraform/files/akvs-certificate-sync.yaml @@ -14,3 +14,4 @@ spec: secret: name: __K8S_TLS_SECRET_NAME__ # kubernetes secret name type: kubernetes.io/tls # kubernetes secret type + chainOrder: ensureserverfirst From c4a5ec9c9c45a8225579bc570b4560e4efa4d941 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 20 Oct 2020 08:22:47 +0100 Subject: [PATCH 29/71] Removed aks_config output --- terraform/outputs.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/outputs.tf b/terraform/outputs.tf index a3411fe..2e8b498 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -1,7 +1,7 @@ # WARNING: this outputs credential / login config -output "aks_config" { - value = azurerm_kubernetes_cluster.aks -} +# output "aks_config" { +# value = azurerm_kubernetes_cluster.aks +# } output "aks_credentials_command" { value = "az aks get-credentials --resource-group ${azurerm_kubernetes_cluster.aks.resource_group_name} --name ${azurerm_kubernetes_cluster.aks.name} --overwrite-existing" From 4a4fe67d1b7bcc48f3efc73d3d80b569ce9d81c0 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 20 Oct 2020 08:25:30 +0100 Subject: [PATCH 30/71] Added nexus_image_tag --- terraform/helm_nexus.tf | 5 +++++ terraform/variables.tf | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/terraform/helm_nexus.tf b/terraform/helm_nexus.tf index cd0f357..9409164 100644 --- a/terraform/helm_nexus.tf +++ b/terraform/helm_nexus.tf @@ -21,6 +21,11 @@ resource "helm_release" "nexus" { version = var.nexus_chart_version values = ["${file("helm/nexus_values.yaml")}"] + set { + name = "image.tag" + value = var.nexus_image_tag + } + set { name = "nexus.baseDomain" value = var.nexus_base_domain diff --git a/terraform/variables.tf b/terraform/variables.tf index 33d3a4a..4dc1373 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -30,6 +30,11 @@ variable "velero_chart_version" { default = "2.13.3" } +# https://hub.docker.com/r/sonatype/nexus3/tags +variable "nexus_image_tag" { + default = "3.28.1" +} + # https://github.com/adamrushuk/charts/releases variable "nexus_chart_version" { default = "0.2.7" From 0c515132794ad6e439dd11969e8ef5b08ae0dc60 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 20 Oct 2020 12:46:53 +0100 Subject: [PATCH 31/71] Added new env var method --- .github/workflows/destroy.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index d0b5d89..88fbd89 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -75,16 +75,18 @@ jobs: # ref: develop # Env var concatenation - # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 + # Old method: https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 + # New method: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files + # echo "action_state=yellow" >> $GITHUB_ENV - name: Concatenate env vars (Workaround) run: | chmod -R +x ./scripts/ - echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 - echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 - echo ::set-env name=TERRAFORM_STORAGE_ACCOUNT::${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001 - echo ::set-env name=TERRAFORM_STORAGE_RG::${{ env.PREFIX }}-rg-tfstate-dev-001 - echo ::set-env name=VELERO_STORAGE_RG::${{ env.PREFIX }}-rg-velero-dev-001 - echo ::set-env name=VELERO_STORAGE_ACCOUNT::${{ env.PREFIX }}stbckuksouth001 + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_ACCOUNT=${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_RG=${{ env.PREFIX }}-rg-tfstate-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_RG=${{ env.PREFIX }}-rg-velero-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_ACCOUNT=${{ env.PREFIX }}stbckuksouth001" >> $GITHUB_ENV # Show event info - name: Show triggered event data From 17cf7d77eb441a5d2504686f6ee1ac2d1634139e Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 20 Oct 2020 12:52:38 +0100 Subject: [PATCH 32/71] Added new env var method to build workflow --- .github/workflows/build.yml | 14 +++++++------- .github/workflows/destroy.yml | 4 +--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b2429f0..c846f10 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,16 +86,16 @@ jobs: # ref: develop # Init tasks - inc Env var concatenation - # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 + # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files - name: Init tasks - inc Env var concatenation (Workaround) run: | chmod -R +x ./scripts/ - echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 - echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 - echo ::set-env name=TERRAFORM_STORAGE_ACCOUNT::${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001 - echo ::set-env name=TERRAFORM_STORAGE_RG::${{ env.PREFIX }}-rg-tfstate-dev-001 - echo ::set-env name=VELERO_STORAGE_RG::${{ env.PREFIX }}-rg-velero-dev-001 - echo ::set-env name=VELERO_STORAGE_ACCOUNT::${{ env.PREFIX }}stbckuksouth001 + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_ACCOUNT=${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_RG=${{ env.PREFIX }}-rg-tfstate-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_RG=${{ env.PREFIX }}-rg-velero-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_ACCOUNT=${{ env.PREFIX }}stbckuksouth001" >> $GITHUB_ENV # Show event info - name: Show triggered event data diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 88fbd89..9ed843a 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -75,9 +75,7 @@ jobs: # ref: develop # Env var concatenation - # Old method: https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 - # New method: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files - # echo "action_state=yellow" >> $GITHUB_ENV + # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files - name: Concatenate env vars (Workaround) run: | chmod -R +x ./scripts/ From 7e92be44f35771c2189bf77afa70fcba1cf3bcdb Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 20 Oct 2020 13:00:57 +0100 Subject: [PATCH 33/71] Bumped velero-plugin-for-microsoft-azure to v1.1.1 --- terraform/helm/velero_values.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/terraform/helm/velero_values.yaml b/terraform/helm/velero_values.yaml index 81acdf8..9a6fa99 100644 --- a/terraform/helm/velero_values.yaml +++ b/terraform/helm/velero_values.yaml @@ -1,4 +1,4 @@ -# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.12.17/charts/velero/values.yaml +# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.13.3/charts/velero/values.yaml ## ## Configuration settings that directly affect the Velero deployment YAML. ## @@ -28,7 +28,6 @@ podAnnotations: {} # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ podLabels: {} - # Resource requests/limits to specify for the Velero deployment. Optional. resources: {} @@ -38,7 +37,7 @@ initContainers: - name: velero-plugin-for-microsoft-azure # use "main" for development build # https://hub.docker.com/r/velero/velero-plugin-for-microsoft-azure/tags - image: velero/velero-plugin-for-microsoft-azure:v1.1.0 + image: velero/velero-plugin-for-microsoft-azure:v1.1.1 # image: velero/velero-plugin-for-microsoft-azure:main imagePullPolicy: IfNotPresent volumeMounts: @@ -93,7 +92,7 @@ configuration: provider: azure # Parameters for the `default` BackupStorageLocation. See - # https://velero.io/docs/v1.4/api-types/backupstoragelocation/ + # https://velero.io/docs/v1.5/api-types/backupstoragelocation/ backupStorageLocation: # name is the name of the backup storage location where backups should be stored. If a name is not provided, # a backup storage location will be created with the name "default". Optional. @@ -124,7 +123,7 @@ configuration: # serviceAccount: # Parameters for the `default` VolumeSnapshotLocation. See - # https://velero.io/docs/v1.4/api-types/volumesnapshotlocation/ + # https://velero.io/docs/v1.5/api-types/volumesnapshotlocation/ volumeSnapshotLocation: # name is the name of the volume snapshot location where snapshots are being taken. Required. name: default @@ -168,6 +167,9 @@ configuration: # Set log-format for Velero pod. Default: text. Other option: json. logFormat: + # Set true for backup all pod volumes without having to apply annotation on the pod when used restic Default: false. Other option: false. + defaultVolumesToRestic: + ## ## End of backup/snapshot location settings. ## @@ -242,13 +244,13 @@ restic: # Eg: # schedules: # mybackup: -# labels: -# myenv: foo +# labels: +# myenv: foo # schedule: "0 0 * * *" # template: # ttl: "240h" # includedNamespaces: -# - foo +# - foo schedules: {} # Velero ConfigMaps. From 9ec35f3d4c7e6a8c8f40b7830e17fbe27c583e78 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 20 Oct 2020 13:41:22 +0100 Subject: [PATCH 34/71] Removed Service Principle for AKS --- terraform/variables.tf | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 4dc1373..1c75067 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -125,17 +125,6 @@ variable "aks_config_path" { } -# TODO DELETE SECTION -# Service Principle for AKS -# variable "service_principal_client_id" { -# default = "__ARM_CLIENT_ID__" -# } - -# variable "service_principal_client_secret" { -# default = "__ARM_CLIENT_SECRET__" -# } -# TODO DELETE SECTION - # Agent Pool variable "agent_pool_node_count" { From dbc7bb13e952c36afaf9eda8a9596248db5816a4 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 20 Oct 2020 13:42:31 +0100 Subject: [PATCH 35/71] Changed velero to disabled --- .github/workflows/build.yml | 2 +- .github/workflows/destroy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c846f10..9d6870a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ env: ROOT_DOMAIN_NAME: thehypepipe.co.uk # NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }} # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: true + VELERO_ENABLED: false # terraform TF_IN_AUTOMATION: "true" diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 9ed843a..82da382 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -45,7 +45,7 @@ env: LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: true + VELERO_ENABLED: false # terraform TF_IN_AUTOMATION: "true" From 18196b06ed25662108352f67b4df1ae9fb60f766 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Wed, 21 Oct 2020 10:07:44 +0100 Subject: [PATCH 36/71] Enabled velero --- .github/workflows/build.yml | 2 +- .github/workflows/destroy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d6870a..c846f10 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ env: ROOT_DOMAIN_NAME: thehypepipe.co.uk # NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }} # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: false + VELERO_ENABLED: true # terraform TF_IN_AUTOMATION: "true" diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 82da382..9ed843a 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -45,7 +45,7 @@ env: LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: false + VELERO_ENABLED: true # terraform TF_IN_AUTOMATION: "true" From 5db7f141cacf36301b7949c7bd7c98cbf4d0ec76 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Wed, 21 Oct 2020 10:08:50 +0100 Subject: [PATCH 37/71] Bumped aad-pod-identity chart --- aad-pod-identity/README.md | 6 ++--- aad-pod-identity/aad_pod_identity_values.yaml | 23 ++++++++++++++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/aad-pod-identity/README.md b/aad-pod-identity/README.md index 0ce7bd9..b368519 100644 --- a/aad-pod-identity/README.md +++ b/aad-pod-identity/README.md @@ -25,7 +25,7 @@ Before automating the installation and configuration of aad-pod-identity, follow ### Deploy aad-pod-identity using Helm 3 -Repo: [https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity](https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity) +Repo: [aad-pod-identity chart](https://github.com/Azure/aad-pod-identity/tree/master/charts/aad-pod-identity) ```bash # Navigate to aad-pod-identity folder @@ -44,7 +44,7 @@ helm search repo aad-pod-identity --version ^2.0.0 kubectl create namespace aad-pod-identity # Install aad-pod-identity -helm upgrade aad-pod-identity aad-pod-identity/aad-pod-identity --version 2.0.1 --values aad_pod_identity_values.yaml --install --atomic --namespace aad-pod-identity --debug +helm upgrade aad-pod-identity aad-pod-identity/aad-pod-identity --version 2.0.2 --values aad_pod_identity_values.yaml --set=installCRDs=true --install --atomic --namespace aad-pod-identity --debug ``` ### Configure aad-pod-identity @@ -116,7 +116,7 @@ kubectl create secret generic --namespace velero velero-credentials --from-file= Once `aad-pod-identity` has been configured, and the Velero credentials secret has been populated, install Velero via Helm chart ensuring the aadpodidbinding=$IDENTITY_NAME label has been added to the Velero values.yaml, eg: ```yaml -# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.12.13/charts/velero/values.yaml#L24 +# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.13.3/charts/velero/values.yaml#L27 podLabels: aadpodidbinding: velero ``` diff --git a/aad-pod-identity/aad_pod_identity_values.yaml b/aad-pod-identity/aad_pod_identity_values.yaml index 4fc2bc9..bc6e53c 100644 --- a/aad-pod-identity/aad_pod_identity_values.yaml +++ b/aad-pod-identity/aad_pod_identity_values.yaml @@ -1,3 +1,5 @@ +# source: https://github.com/Azure/aad-pod-identity/blob/v1.6.3/charts/aad-pod-identity/values.yaml + # Default values for aad-pod-identity-helm. # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -6,9 +8,13 @@ nameOverride: "" fullnameOverride: "" image: - repository: mcr.microsoft.com/k8s/aad-pod-identity + repository: mcr.microsoft.com/oss/azure/aad-pod-identity imagePullPolicy: Always +# One or more secrets to be used when pulling images +# imagePullSecrets: +# - name: myRegistryKeySecretName + # https://github.com/Azure/aad-pod-identity#4-optional-match-pods-in-the-namespace # By default, AAD Pod Identity matches pods to identities across namespaces. # To match only pods in the namespace containing AzureIdentity set this to true. @@ -29,16 +35,17 @@ adminsecret: {} # resourceGroup: # vmType: <`standard` for normal virtual machine nodes, and `vmss` for cluster deployed with a virtual machine scale set> # tenantID: -# clientID: -# clientSecret: - +# clientID: +# clientSecret: +# useMSI: +# userAssignedMSIClientID: # Operation mode for pod-identity. Default is standard mode that has MIC doing identity assignment # Allowed values: "standard", "managed" operationMode: "standard" mic: image: mic - tag: 1.6.2 + tag: v1.6.3 priorityClassName: "" @@ -111,9 +118,13 @@ mic: # Default value is 1s updateUserMSIRetryInterval: "" + # The interval between reconciling identity assignment on Azure based on an existing list of AzureAssignedIdentities + # Default value is 3m + identityAssignmentReconcileInterval: "" + nmi: image: nmi - tag: 1.6.2 + tag: v1.6.3 priorityClassName: "" From f1b09d6e036c20fd6b1d38a709e66f61cc1d294c Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Wed, 21 Oct 2020 10:09:26 +0100 Subject: [PATCH 38/71] Changed velero config for aadpodidbinding --- terraform/helm/velero_values.yaml | 2 -- terraform/velero.tf | 9 ++++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/terraform/helm/velero_values.yaml b/terraform/helm/velero_values.yaml index 9a6fa99..581bd1d 100644 --- a/terraform/helm/velero_values.yaml +++ b/terraform/helm/velero_values.yaml @@ -35,10 +35,8 @@ resources: {} # https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/releases initContainers: - name: velero-plugin-for-microsoft-azure - # use "main" for development build # https://hub.docker.com/r/velero/velero-plugin-for-microsoft-azure/tags image: velero/velero-plugin-for-microsoft-azure:v1.1.1 - # image: velero/velero-plugin-for-microsoft-azure:main imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /target diff --git a/terraform/velero.tf b/terraform/velero.tf index f20b8bf..e4d75eb 100644 --- a/terraform/velero.tf +++ b/terraform/velero.tf @@ -69,9 +69,6 @@ resource "kubernetes_secret" "velero_credentials" { data = { cloud = < Date: Wed, 21 Oct 2020 13:42:05 +0100 Subject: [PATCH 39/71] Added MSI role assignment for AKS node RG --- aad-pod-identity/README.md | 16 +++++++--- terraform/velero_auth.tf | 59 ++++++++++++++++++------------------ terraform/velero_msi_auth.tf | 7 +++++ 3 files changed, 49 insertions(+), 33 deletions(-) create mode 100644 terraform/velero_msi_auth.tf diff --git a/aad-pod-identity/README.md b/aad-pod-identity/README.md index b368519..0eb1641 100644 --- a/aad-pod-identity/README.md +++ b/aad-pod-identity/README.md @@ -73,6 +73,14 @@ echo "IDENTITY_CLIENT_ID: $IDENTITY_CLIENT_ID" # Assign the identity a role export IDENTITY_ASSIGNMENT_ID="$(az role assignment create --role Contributor --assignee "$IDENTITY_CLIENT_ID" --scope "$AKS_NODE_RESOURCE_GROUP_ID" --query id -o tsv)" +# Describe AzureIdentity CRDs (they dont have metadata) +kubectl describe AzureIdentity velero +kubectl explain --recursive AzureIdentity +kubectl explain --recursive backups +kubectl explain --recursive AzureIdentity.spec +kubectl explain --recursive AzureIdentityBinding +kubectl explain --recursive AzureIdentityBinding.spec + # Create an AzureIdentity cat < Date: Wed, 21 Oct 2020 14:45:35 +0100 Subject: [PATCH 40/71] Disabled velero --- .github/workflows/build.yml | 2 +- .github/workflows/destroy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c846f10..9d6870a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ env: ROOT_DOMAIN_NAME: thehypepipe.co.uk # NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }} # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: true + VELERO_ENABLED: false # terraform TF_IN_AUTOMATION: "true" diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 9ed843a..82da382 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -45,7 +45,7 @@ env: LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: true + VELERO_ENABLED: false # terraform TF_IN_AUTOMATION: "true" From 3b00af1f245140d77e7441da08accff31e202912 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Wed, 21 Oct 2020 14:46:06 +0100 Subject: [PATCH 41/71] Added aad-pod-identity helm release --- terraform/helm/aad_pod_identity_values.yaml | 206 ++++++++++++++++++++ terraform/helm_aad_pod_identity.tf | 31 +++ terraform/variables.tf | 5 + 3 files changed, 242 insertions(+) create mode 100644 terraform/helm/aad_pod_identity_values.yaml create mode 100644 terraform/helm_aad_pod_identity.tf diff --git a/terraform/helm/aad_pod_identity_values.yaml b/terraform/helm/aad_pod_identity_values.yaml new file mode 100644 index 0000000..d643e1b --- /dev/null +++ b/terraform/helm/aad_pod_identity_values.yaml @@ -0,0 +1,206 @@ +# source: https://github.com/Azure/aad-pod-identity/blob/v1.6.3/charts/aad-pod-identity/values.yaml + +# Default values for aad-pod-identity-helm. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +nameOverride: "" +fullnameOverride: "" + +image: + repository: mcr.microsoft.com/oss/azure/aad-pod-identity + imagePullPolicy: Always + +# One or more secrets to be used when pulling images +# imagePullSecrets: +# - name: myRegistryKeySecretName + +# https://github.com/Azure/aad-pod-identity#4-optional-match-pods-in-the-namespace +# By default, AAD Pod Identity matches pods to identities across namespaces. +# To match only pods in the namespace containing AzureIdentity set this to true. +forceNameSpaced: "false" + +# When NMI runs on a node where MIC is running, then MIC token request call is also +# intercepted by NMI. MIC can't get a valid token as to initialize and then +# assign the identity. Installing an exception for MIC would ensure all token requests +# for MIC pods directly go to IMDS and not go through the pod-identity validation +# https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.app-exception.md +installMICException: "true" + +## If using a separate service principal for aad-pod-identity instead of cluster service principal specify the following +## (The chart will perform the base64 encoding for you for values that are stored in secrets.) +adminsecret: {} +# cloud: +# subscriptionID: +# resourceGroup: +# vmType: <`standard` for normal virtual machine nodes, and `vmss` for cluster deployed with a virtual machine scale set> +# tenantID: +# clientID: +# clientSecret: +# useMSI: +# userAssignedMSIClientID: +# Operation mode for pod-identity. Default is standard mode that has MIC doing identity assignment +# Allowed values: "standard", "managed" +operationMode: "standard" + +mic: + image: mic + tag: v1.6.3 + + priorityClassName: "" + + # log level. Uses V logs (glog) + logVerbosity: 0 + + resources: + limits: + cpu: 200m + memory: 1024Mi + requests: + cpu: 100m + memory: 256Mi + + podAnnotations: {} + + ## Node labels for pod assignment + ## aad-pod-identity is currently only supported on linux + nodeSelector: + kubernetes.io/os: linux + + tolerations: [] + + affinity: {} + + leaderElection: + # Override leader election instance name (default is 'hostname') + instance: "" + # Override the namespace to create leader election objects (default is default namespace) + namespace: "" + # Override leader election name (default is aad-pod-identity-mic) + name: "" + # Override leader election duration (default is 15s) + duration: "" + + # Override http liveliness probe port (default is 8080) + probePort: "" + + # Override interval in seconds at which sync loop should periodically check for errors and reconcile (default is 3600s) + syncRetryDuration: "" + + # Override the defult value of immutable identities. + immutableUserMSIs: [] + # Example of MSIs (should be replaced with the real client ids) + #- "00000000-0000-0000-0000-000000000000" + #- "11111111-1111-1111-1111-111111111111" + + # https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#batch-create-delete-flag + # default value is 20 + createDeleteBatch: "" + + # https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#client-qps-flag + # default value is 5 + clientQps: "" + + # default value is 8888 + # prometheus port for metrics + prometheusPort: "" + + # cloud configuration used to authenticate with Azure + cloudConfig: "/etc/kubernetes/azure.json" + + # The maximum retry of UpdateUserMSI call. MIC updates all the identities in a batch. If a single identity contains an error + # or is invalid, then the entire operation fails. Configuring this flag will make MIC retry by removing the erroneous identities + # returned in the error + # Default value is 2. + updateUserMSIMaxRetry: "" + + # The duration to wait before retrying UpdateUserMSI (batch assigning/un-assigning identity from VM/VMSS) in case of errors + # Default value is 1s + updateUserMSIRetryInterval: "" + + # The interval between reconciling identity assignment on Azure based on an existing list of AzureAssignedIdentities + # Default value is 3m + identityAssignmentReconcileInterval: "" + +nmi: + image: nmi + tag: v1.6.3 + + priorityClassName: "" + + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + + podAnnotations: {} + + ## Node labels for pod assignment + ## aad-pod-identity is currently only supported on linux + nodeSelector: + kubernetes.io/os: linux + + tolerations: [] + + affinity: {} + + # Override iptables update interval in seconds (default is 60) + ipTableUpdateTimeIntervalInSeconds: "" + + # Override mic namespace to short circuit MIC token requests (default is default namespace) + micNamespace: "" + + # Override http liveliness probe port (default is 8080) + probePort: "8085" + + # Override number of retries in NMI to find assigned identity in CREATED state (default is 16) + retryAttemptsForCreated: "" + + # Override number of retries in NMI to find assigned identity in ASSIGNED state (default is 4) + retryAttemptsForAssigned: "" + + # Override retry interval to find assigned identities in seconds (default is 5) + findIdentityRetryIntervalInSeconds: "" + + # Enable scale features - https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#enable-scale-features-flag + # Accepted values are true/false. Default is false. + enableScaleFeatures: "" + + # default value is 9090 + # prometheus port for metrics + prometheusPort: "" + + # https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#block-instance-metadata-flag + # default is false + blockInstanceMetadata: "" + + # https://github.com/Azure/aad-pod-identity/blob/master/docs/readmes/README.featureflags.md#metadata-header-required-flag + # default is false + metadataHeaderRequired: "" + +rbac: + enabled: true + # NMI requires permissions to get secrets when service principal (type: 1) is used in AzureIdentity. + # If using only MSI (type: 0) in AzureIdentity, secret get permission can be disabled by setting this to false. + allowAccessToSecrets: true + +# Create azure identities and bindings +azureIdentities: + - name: "velero" + # if not defined, then the azure identity will be deployed in the same namespace as the chart + namespace: "" + # type 0: MSI, type 1: Service Principal + type: 0 + # /subscriptions/subscription-id/resourcegroups/resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/identity-name + resourceID: "" + clientID: "" + binding: + name: "velero-binding" + # The selector will also need to be included in labels for app deployment + selector: "velero" + +# If true, install necessary custom resources. +installCRDs: false diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf new file mode 100644 index 0000000..e007015 --- /dev/null +++ b/terraform/helm_aad_pod_identity.tf @@ -0,0 +1,31 @@ +# aad-pod-identity helm chart + +# https://www.terraform.io/docs/providers/kubernetes/r/namespace.html +resource "kubernetes_namespace" "aad_pod_identity" { + metadata { + name = "aad-pod-identity" + } + timeouts { + delete = "15m" + } + + depends_on = [azurerm_kubernetes_cluster.aks] +} + +# https://www.terraform.io/docs/providers/helm/r/release.html +resource "helm_release" "aad_pod_identity" { + chart = "aad-pod-identity" + name = "aad-pod-identity" + namespace = "aad-pod-identity" + repository = "https://raw.githubusercontent.com/Azure/aad-pod-identity/master/charts" + version = var.aad_pod_identity_chart_version + values = ["${file("helm/aad_pod_identity_values.yaml")}"] + + set { + name = "installCRDs" + value = "true" + } + + timeout = 600 + depends_on = [kubernetes_namespace.aad_pod_identity] +} diff --git a/terraform/variables.tf b/terraform/variables.tf index 1c75067..3c53272 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -46,6 +46,11 @@ variable "nexus_chart_version" { variable "akv2k8s_chart_version" { default = "1.1.25" } + +# https://github.com/Azure/aad-pod-identity/blob/master/charts/aad-pod-identity/Chart.yaml#L4 +variable "aad_pod_identity_chart_version" { + default = "2.0.2" +} #endregion Versions From 471e56e62078dbe19f16c0c18596ce6e19372608 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Wed, 21 Oct 2020 19:17:29 +0100 Subject: [PATCH 42/71] Removed azureIdentities --- terraform/helm/aad_pod_identity_values.yaml | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/terraform/helm/aad_pod_identity_values.yaml b/terraform/helm/aad_pod_identity_values.yaml index d643e1b..cbc226f 100644 --- a/terraform/helm/aad_pod_identity_values.yaml +++ b/terraform/helm/aad_pod_identity_values.yaml @@ -188,19 +188,19 @@ rbac: allowAccessToSecrets: true # Create azure identities and bindings -azureIdentities: - - name: "velero" - # if not defined, then the azure identity will be deployed in the same namespace as the chart - namespace: "" - # type 0: MSI, type 1: Service Principal - type: 0 - # /subscriptions/subscription-id/resourcegroups/resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/identity-name - resourceID: "" - clientID: "" - binding: - name: "velero-binding" - # The selector will also need to be included in labels for app deployment - selector: "velero" +azureIdentities: [] + # - name: "velero" + # # if not defined, then the azure identity will be deployed in the same namespace as the chart + # namespace: "" + # # type 0: MSI, type 1: Service Principal + # type: 0 + # # /subscriptions/subscription-id/resourcegroups/resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/identity-name + # resourceID: "" + # clientID: "" + # binding: + # name: "velero-binding" + # # The selector will also need to be included in labels for app deployment + # selector: "velero" # If true, install necessary custom resources. installCRDs: false From d72db85fc174f5b1c6b69d7938399b6ff724f656 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Wed, 21 Oct 2020 19:17:41 +0100 Subject: [PATCH 43/71] Bumped velero chart to v2.13.6 --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 3c53272..0b0805f 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -27,7 +27,7 @@ variable "cert_manager_chart_version" { # https://github.com/vmware-tanzu/helm-charts/releases variable "velero_chart_version" { - default = "2.13.3" + default = "2.13.6" } # https://hub.docker.com/r/sonatype/nexus3/tags From 209ca62a45131dd5c4955ff887cafb5d237dffba Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 08:02:31 +0100 Subject: [PATCH 44/71] Enabled velero --- .github/workflows/build.yml | 2 +- .github/workflows/destroy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d6870a..c846f10 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ env: ROOT_DOMAIN_NAME: thehypepipe.co.uk # NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }} # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: false + VELERO_ENABLED: true # terraform TF_IN_AUTOMATION: "true" diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 82da382..9ed843a 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -45,7 +45,7 @@ env: LOCATION: uksouth ROOT_DOMAIN_NAME: thehypepipe.co.uk # STORAGE_KEY: 'env var set by Get-StorageKey.ps1' - VELERO_ENABLED: false + VELERO_ENABLED: true # terraform TF_IN_AUTOMATION: "true" From 0ecdf53d089d9a13b1b251cf38abe726a1e3b385 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 08:03:10 +0100 Subject: [PATCH 45/71] Added Velero MI and azureIdentities yaml template --- terraform/helm/azureIdentities.yaml.tpl | 13 +++++++++++++ terraform/helm_aad_pod_identity.tf | 26 ++++++++++++++++++++++++- terraform/velero_auth.tf | 2 +- terraform/velero_mi_auth.tf | 16 +++++++++++++++ terraform/velero_msi_auth.tf | 7 ------- 5 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 terraform/helm/azureIdentities.yaml.tpl create mode 100644 terraform/velero_mi_auth.tf delete mode 100644 terraform/velero_msi_auth.tf diff --git a/terraform/helm/azureIdentities.yaml.tpl b/terraform/helm/azureIdentities.yaml.tpl new file mode 100644 index 0000000..a37dcce --- /dev/null +++ b/terraform/helm/azureIdentities.yaml.tpl @@ -0,0 +1,13 @@ +azureIdentities: + - name: "velero" + # if not defined, then the azure identity will be deployed in the same namespace as the chart + namespace: "" + # type 0: MSI, type 1: Service Principal + type: 0 + # /subscriptions/subscription-id/resourcegroups/resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/identity-name + resourceID: "${resourceID}" + clientID: "${clientID}" + binding: + name: "velero-binding" + # The selector will also need to be included in labels for app deployment + selector: "velero" diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index e007015..82cc3c8 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -1,7 +1,26 @@ # aad-pod-identity helm chart +# role assignment for aad-pod-identity +resource "azurerm_role_assignment" "aks_mi_aks_node_rg" { + count = var.velero_enabled ? 1 : 0 + principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + role_definition_name = "Contributor" + scope = data.azurerm_resource_group.aks.id + skip_service_principal_aad_check = true +} + +data "template_file" "azureIdentities" { + count = var.velero_enabled ? 1 : 0 + template = "${file("${path.module}/helm/azureIdentities.yaml.tpl")}" + vars = { + resourceID = azurerm_user_assigned_identity.velero.id + clientID = azurerm_user_assigned_identity.velero.client_id + } +} + # https://www.terraform.io/docs/providers/kubernetes/r/namespace.html resource "kubernetes_namespace" "aad_pod_identity" { + count = var.velero_enabled ? 1 : 0 metadata { name = "aad-pod-identity" } @@ -14,12 +33,17 @@ resource "kubernetes_namespace" "aad_pod_identity" { # https://www.terraform.io/docs/providers/helm/r/release.html resource "helm_release" "aad_pod_identity" { + count = var.velero_enabled ? 1 : 0 chart = "aad-pod-identity" name = "aad-pod-identity" namespace = "aad-pod-identity" repository = "https://raw.githubusercontent.com/Azure/aad-pod-identity/master/charts" version = var.aad_pod_identity_chart_version - values = ["${file("helm/aad_pod_identity_values.yaml")}"] + + values = [ + "${file("helm/aad_pod_identity_values.yaml")}" + data.template_file.azureIdentities.rendered + ] set { name = "installCRDs" diff --git a/terraform/velero_auth.tf b/terraform/velero_auth.tf index b1a7b8a..da621b8 100644 --- a/terraform/velero_auth.tf +++ b/terraform/velero_auth.tf @@ -1,4 +1,4 @@ -# TODO remove this after confirming MSI auth works +# TODO remove this after confirming MI auth works # Service Principle for velero # resource "azuread_application" "velero_sp" { # name = var.velero_service_principle_name diff --git a/terraform/velero_mi_auth.tf b/terraform/velero_mi_auth.tf new file mode 100644 index 0000000..6c07d47 --- /dev/null +++ b/terraform/velero_mi_auth.tf @@ -0,0 +1,16 @@ +# velero managed identity auth +resource "azurerm_user_assigned_identity" "velero" { + count = var.velero_enabled ? 1 : 0 + resource_group_name = azurerm_resource_group.velero[0].name + location = azurerm_resource_group.velero[0].location + + name = "mi_velero" +} + +# assign velero MI contributor rights to velero storage RG +resource "azurerm_role_assignment" "velero_mi_velero_storage_rg" { + count = var.velero_enabled ? 1 : 0 + principal_id = azurerm_user_assigned_identity.velero.id + role_definition_name = "Contributor" + scope = azurerm_resource_group.velero.id +} diff --git a/terraform/velero_msi_auth.tf b/terraform/velero_msi_auth.tf deleted file mode 100644 index 590bc31..0000000 --- a/terraform/velero_msi_auth.tf +++ /dev/null @@ -1,7 +0,0 @@ -# role assignment for aad-pod-identity / velero auth -resource "azurerm_role_assignment" "aks_msi_aks_node_rg" { - principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id - role_definition_name = "Contributor" - scope = data.azurerm_resource_group.aks.id - skip_service_principal_aad_check = true -} From 57c5b252bbba71e47b2cc9c44f153837d99d7560 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 08:08:45 +0100 Subject: [PATCH 46/71] Fixed missing comma for helm values list --- terraform/helm_aad_pod_identity.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index 82cc3c8..d902dcc 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -41,7 +41,7 @@ resource "helm_release" "aad_pod_identity" { version = var.aad_pod_identity_chart_version values = [ - "${file("helm/aad_pod_identity_values.yaml")}" + "${file("helm/aad_pod_identity_values.yaml")}", data.template_file.azureIdentities.rendered ] From 3928d185dd2f2d890ff4a634a810b4ae73891f24 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 08:17:16 +0100 Subject: [PATCH 47/71] Added missing count indices --- terraform/helm_aad_pod_identity.tf | 4 ++-- terraform/velero_mi_auth.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index d902dcc..1d4f4de 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -13,8 +13,8 @@ data "template_file" "azureIdentities" { count = var.velero_enabled ? 1 : 0 template = "${file("${path.module}/helm/azureIdentities.yaml.tpl")}" vars = { - resourceID = azurerm_user_assigned_identity.velero.id - clientID = azurerm_user_assigned_identity.velero.client_id + resourceID = azurerm_user_assigned_identity.velero[0].id + clientID = azurerm_user_assigned_identity.velero[0].client_id } } diff --git a/terraform/velero_mi_auth.tf b/terraform/velero_mi_auth.tf index 6c07d47..578bf11 100644 --- a/terraform/velero_mi_auth.tf +++ b/terraform/velero_mi_auth.tf @@ -10,7 +10,7 @@ resource "azurerm_user_assigned_identity" "velero" { # assign velero MI contributor rights to velero storage RG resource "azurerm_role_assignment" "velero_mi_velero_storage_rg" { count = var.velero_enabled ? 1 : 0 - principal_id = azurerm_user_assigned_identity.velero.id + principal_id = azurerm_user_assigned_identity.velero[0].id role_definition_name = "Contributor" - scope = azurerm_resource_group.velero.id + scope = azurerm_resource_group.velero[0].id } From a47ce0bccb3b82e428ced72c786ffde891eb7bfc Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 08:23:23 +0100 Subject: [PATCH 48/71] Added missing array indices --- terraform/helm_aad_pod_identity.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index 1d4f4de..1241cf9 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -42,7 +42,7 @@ resource "helm_release" "aad_pod_identity" { values = [ "${file("helm/aad_pod_identity_values.yaml")}", - data.template_file.azureIdentities.rendered + data.template_file.azureIdentities[0].rendered ] set { @@ -51,5 +51,5 @@ resource "helm_release" "aad_pod_identity" { } timeout = 600 - depends_on = [kubernetes_namespace.aad_pod_identity] + depends_on = [kubernetes_namespace.aad_pod_identity[0]] } From 2accd9a972b1c8ad5f24b2ae53bc015d46a06bc1 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 08:28:53 +0100 Subject: [PATCH 49/71] Changed env vars to new method --- .github/workflows/build.yml | 4 ++-- .github/workflows/deallocate_aks_vmss.yml | 5 ++--- .github/workflows/start_aks_vmss.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- scripts/storage_key.sh | 6 +++--- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c846f10..63fddfa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -261,8 +261,8 @@ jobs: # Init tasks - inc Env var concatenation - name: Init tasks - inc Env var concatenation (Workaround) run: | - echo "::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001" - echo "::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001" + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV # Login - name: Login to Azure diff --git a/.github/workflows/deallocate_aks_vmss.yml b/.github/workflows/deallocate_aks_vmss.yml index 01195d9..161e4d3 100644 --- a/.github/workflows/deallocate_aks_vmss.yml +++ b/.github/workflows/deallocate_aks_vmss.yml @@ -43,9 +43,8 @@ jobs: - name: Init tasks - inc Env var concatenation (Workaround) run: | chmod -R +x ./scripts/ - echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 - echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 - + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV # Show event info - name: Show triggered event data run: pwsh -command "./scripts/Get-EventData.ps1" diff --git a/.github/workflows/start_aks_vmss.yml b/.github/workflows/start_aks_vmss.yml index a64dafb..04d7d12 100644 --- a/.github/workflows/start_aks_vmss.yml +++ b/.github/workflows/start_aks_vmss.yml @@ -43,8 +43,8 @@ jobs: - name: Init tasks - inc Env var concatenation (Workaround) run: | chmod -R +x ./scripts/ - echo ::set-env name=AKS_RG_NAME::${{ env.PREFIX }}-rg-aks-dev-001 - echo ::set-env name=AKS_CLUSTER_NAME::${{ env.PREFIX }}-aks-001 + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV # Show event info - name: Show triggered event data diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1c9790..924fc7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,7 @@ jobs: # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 - name: BASH - Concatenate env vars (Workaround) run: | - echo ::set-env name=MY_CONCATENATED_BASH_VAR::${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-bash-stepvalue01 + echo "MY_CONCATENATED_BASH_VAR=${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-bash-stepvalue01 >> $GITHUB_ENV - name: PWSH - Concatenate env vars (Workaround) # override the default bash shell, as running on ubuntu @@ -66,7 +66,7 @@ jobs: shell: pwsh # pwsh requires double quotes when setting env vars run: | - echo "::set-env name=MY_CONCATENATED_PWSH_VAR::${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-pwsh-stepvalue01" + echo ""MY_CONCATENATED_PWSH_VAR=${{ env.MY_WORKFLOW_VAR }}-${{ env.MY_JOB_VAR }}-pwsh-stepvalue01" >> $GITHUB_ENV - name: Output event data shell: pwsh diff --git a/scripts/storage_key.sh b/scripts/storage_key.sh index 558a4e1..1b2a20a 100644 --- a/scripts/storage_key.sh +++ b/scripts/storage_key.sh @@ -17,9 +17,9 @@ echo "FINISHED: $taskMessage." taskMessage="Updating workflow env vars" echo "STARTED: $taskMessage..." -# https://help.github.com/en/actions/reference/development-tools-for-github-actions#set-an-environment-variable-set-env -# ::set-env name={name}::{value} -echo "::set-env name=STORAGE_KEY::$storage_key" +# # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files +# echo "{name}={value}" >> $GITHUB_ENV +echo "STORAGE_KEY=$storage_key" >> "$GITHUB_ENV" # Mask sensitive env var # https://help.github.com/en/actions/reference/development-tools-for-github-actions#example-masking-an-environment-variable From 2f0bbb2dc020711380c801503f8eee7732c220c6 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 08:30:29 +0100 Subject: [PATCH 50/71] Fixed terraform_deprecated_interpolation error --- terraform/helm_aad_pod_identity.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index 1241cf9..1208c23 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -41,7 +41,7 @@ resource "helm_release" "aad_pod_identity" { version = var.aad_pod_identity_chart_version values = [ - "${file("helm/aad_pod_identity_values.yaml")}", + file("helm/aad_pod_identity_values.yaml"), data.template_file.azureIdentities[0].rendered ] From c992e6f42bef2749eb936c81eb2bddde4a31f7c0 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 08:41:19 +0100 Subject: [PATCH 51/71] Fixed another terraform_deprecated_interpolation error --- terraform/helm_aad_pod_identity.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index 1208c23..f29fc51 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -11,7 +11,7 @@ resource "azurerm_role_assignment" "aks_mi_aks_node_rg" { data "template_file" "azureIdentities" { count = var.velero_enabled ? 1 : 0 - template = "${file("${path.module}/helm/azureIdentities.yaml.tpl")}" + template = file("${path.module}/helm/azureIdentities.yaml.tpl") vars = { resourceID = azurerm_user_assigned_identity.velero[0].id clientID = azurerm_user_assigned_identity.velero[0].client_id From 0c8ce2e644a8b488506b5933794f0e75e067a945 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 22 Oct 2020 09:00:22 +0100 Subject: [PATCH 52/71] Changed azurerm_user_assigned_identity principal_id --- terraform/velero_mi_auth.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/velero_mi_auth.tf b/terraform/velero_mi_auth.tf index 578bf11..b5174d0 100644 --- a/terraform/velero_mi_auth.tf +++ b/terraform/velero_mi_auth.tf @@ -10,7 +10,7 @@ resource "azurerm_user_assigned_identity" "velero" { # assign velero MI contributor rights to velero storage RG resource "azurerm_role_assignment" "velero_mi_velero_storage_rg" { count = var.velero_enabled ? 1 : 0 - principal_id = azurerm_user_assigned_identity.velero[0].id + principal_id = azurerm_user_assigned_identity.velero[0].principal_id role_definition_name = "Contributor" scope = azurerm_resource_group.velero[0].id } From bdd0cf8b7bd216f73ccdbd9fc8e9c1ead343b0cd Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 23 Oct 2020 07:34:52 +0100 Subject: [PATCH 53/71] Bumped azurerm to v2.33.0, akv2k8s to v1.1.26 --- terraform/providers.tf | 2 +- terraform/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/providers.tf b/terraform/providers.tf index 376a756..7ab3e16 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -30,7 +30,7 @@ terraform { # must include blank features block # https://github.com/terraform-providers/terraform-provider-azurerm/releases provider "azurerm" { - version = "2.32.0" + version = "2.33.0" features {} } diff --git a/terraform/variables.tf b/terraform/variables.tf index 0b0805f..0c323a1 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -44,7 +44,7 @@ variable "nexus_chart_version" { # https://github.com/SparebankenVest/helm-charts/tree/gh-pages/akv2k8s # https://github.com/SparebankenVest/public-helm-charts/blob/master/stable/akv2k8s/Chart.yaml#L5 variable "akv2k8s_chart_version" { - default = "1.1.25" + default = "1.1.26" } # https://github.com/Azure/aad-pod-identity/blob/master/charts/aad-pod-identity/Chart.yaml#L4 From 605fce6f583cfa4e8b96d0f3f649c88df825c4a2 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 23 Oct 2020 08:35:36 +0100 Subject: [PATCH 54/71] Reverted akv2k8s to v1.1.25 --- terraform/aks.tf | 26 -------------------------- terraform/helm_akv2k8s.tf | 26 ++++++++++++++++++++++++++ terraform/variables.tf | 2 +- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/terraform/aks.tf b/terraform/aks.tf index b492431..a4ec96a 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -123,29 +123,3 @@ resource "azurerm_kubernetes_cluster" "aks" { ] } } - - -# Key vault access policy for AKS / akv2k8s -data "azurerm_key_vault" "kv" { - name = var.key_vault_name - resource_group_name = var.key_vault_resource_group_name -} - -resource "azurerm_key_vault_access_policy" "aks" { - key_vault_id = data.azurerm_key_vault.kv.id - - tenant_id = data.azurerm_subscription.current.tenant_id - object_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id - - certificate_permissions = [ - "get" - ] - - key_permissions = [ - "get" - ] - - secret_permissions = [ - "get" - ] -} diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index edfb1d0..165cfe4 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -4,6 +4,32 @@ # https://akv2k8s.io/ # https://github.com/SparebankenVest/azure-key-vault-to-kubernetes +# Key vault access policy for AKS / akv2k8s +data "azurerm_key_vault" "kv" { + name = var.key_vault_name + resource_group_name = var.key_vault_resource_group_name +} + +resource "azurerm_key_vault_access_policy" "aks" { + key_vault_id = data.azurerm_key_vault.kv.id + + tenant_id = data.azurerm_subscription.current.tenant_id + object_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + + certificate_permissions = [ + "get" + ] + + key_permissions = [ + "get" + ] + + secret_permissions = [ + "get" + ] +} + + resource "local_file" "kubeconfig" { sensitive_content = azurerm_kubernetes_cluster.aks.kube_config_raw filename = var.aks_config_path diff --git a/terraform/variables.tf b/terraform/variables.tf index 0c323a1..0b0805f 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -44,7 +44,7 @@ variable "nexus_chart_version" { # https://github.com/SparebankenVest/helm-charts/tree/gh-pages/akv2k8s # https://github.com/SparebankenVest/public-helm-charts/blob/master/stable/akv2k8s/Chart.yaml#L5 variable "akv2k8s_chart_version" { - default = "1.1.26" + default = "1.1.25" } # https://github.com/Azure/aad-pod-identity/blob/master/charts/aad-pod-identity/Chart.yaml#L4 From 01876011f98cf150bfcb79c070b59e5a08478b10 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 24 Oct 2020 07:27:48 +0100 Subject: [PATCH 55/71] Reverted azurerm to v2.32.0 --- terraform/providers.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/providers.tf b/terraform/providers.tf index 7ab3e16..376a756 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -30,7 +30,7 @@ terraform { # must include blank features block # https://github.com/terraform-providers/terraform-provider-azurerm/releases provider "azurerm" { - version = "2.33.0" + version = "2.32.0" features {} } From 660262c98fa4c66b0ce698e79534c29b472587a8 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 24 Oct 2020 17:07:39 +0100 Subject: [PATCH 56/71] Moved azureIdentities yaml template --- terraform/{helm => files}/azureIdentities.yaml.tpl | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename terraform/{helm => files}/azureIdentities.yaml.tpl (100%) diff --git a/terraform/helm/azureIdentities.yaml.tpl b/terraform/files/azureIdentities.yaml.tpl similarity index 100% rename from terraform/helm/azureIdentities.yaml.tpl rename to terraform/files/azureIdentities.yaml.tpl From 8bd1cf2a3fb1a2ceeabd10f9d7818aa2c24e8bab Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 24 Oct 2020 17:08:44 +0100 Subject: [PATCH 57/71] Added comments --- aad-pod-identity/README.md | 3 +++ terraform/files/AzureKeyVaultSecret.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/aad-pod-identity/README.md b/aad-pod-identity/README.md index 0eb1641..8d134b1 100644 --- a/aad-pod-identity/README.md +++ b/aad-pod-identity/README.md @@ -70,6 +70,9 @@ export IDENTITY_RESOURCE_ID="$(az identity show -g "$AKS_NODE_RESOURCE_GROUP_NAM echo "IDENTITY_RESOURCE_ID: $IDENTITY_RESOURCE_ID" echo "IDENTITY_CLIENT_ID: $IDENTITY_CLIENT_ID" +# [OPTIONAL] Check assigned VMSS identity +az vmss identity show -g "$AKS_NODE_RESOURCE_GROUP_NAME" -n "aks-default-39636823-vmss" + # Assign the identity a role export IDENTITY_ASSIGNMENT_ID="$(az role assignment create --role Contributor --assignee "$IDENTITY_CLIENT_ID" --scope "$AKS_NODE_RESOURCE_GROUP_ID" --query id -o tsv)" diff --git a/terraform/files/AzureKeyVaultSecret.yaml b/terraform/files/AzureKeyVaultSecret.yaml index fe562da..b3f5b40 100644 --- a/terraform/files/AzureKeyVaultSecret.yaml +++ b/terraform/files/AzureKeyVaultSecret.yaml @@ -1,4 +1,6 @@ # https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml +# +# https://akv2k8s.io/reference/azure-key-vault-secret/ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: From 587c80f58c08737782c682e14f9b92a8eb1e5145 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 24 Oct 2020 17:12:34 +0100 Subject: [PATCH 58/71] Disabled akv2k8s_yaml_path usage --- terraform/helm_akv2k8s.tf | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 165cfe4..2e323a5 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -32,7 +32,7 @@ resource "azurerm_key_vault_access_policy" "aks" { resource "local_file" "kubeconfig" { sensitive_content = azurerm_kubernetes_cluster.aks.kube_config_raw - filename = var.aks_config_path + filename = var.aks_config_path depends_on = [azurerm_kubernetes_cluster.aks] } @@ -41,15 +41,17 @@ resource "local_file" "kubeconfig" { resource "null_resource" "akv2k8s_crds" { triggers = { # always_run = "${timestamp()}" - akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) + akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) cert_sync_yaml_contents = filemd5(var.cert_sync_yaml_path) } provisioner "local-exec" { interpreter = ["/bin/bash", "-c"] - command = < Date: Sat, 24 Oct 2020 17:12:59 +0100 Subject: [PATCH 59/71] Fixed aad-pod-identity role assignments --- terraform/helm_aad_pod_identity.tf | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index f29fc51..218922b 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -1,17 +1,35 @@ # aad-pod-identity helm chart # role assignment for aad-pod-identity -resource "azurerm_role_assignment" "aks_mi_aks_node_rg" { +# https://azure.github.io/aad-pod-identity/docs/getting-started/role-assignment/#performing-role-assignments +resource "azurerm_role_assignment" "aks_mi_aks_node_rg_vm_contributor" { count = var.velero_enabled ? 1 : 0 principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id - role_definition_name = "Contributor" + role_definition_name = "Virtual Machine Contributor" scope = data.azurerm_resource_group.aks.id skip_service_principal_aad_check = true } +resource "azurerm_role_assignment" "aks_mi_aks_node_rg_mi_operator" { + count = var.velero_enabled ? 1 : 0 + principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + role_definition_name = "Managed Identity Operator" + scope = data.azurerm_resource_group.aks.id + skip_service_principal_aad_check = true +} + +# velero user MI in different RG, so assign role there too +resource "azurerm_role_assignment" "aks_mi_velero_rg_mi_operator" { + count = var.velero_enabled ? 1 : 0 + principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id + role_definition_name = "Managed Identity Operator" + scope = azurerm_user_assigned_identity.velero[0].resource_group_name + skip_service_principal_aad_check = true +} + data "template_file" "azureIdentities" { count = var.velero_enabled ? 1 : 0 - template = file("${path.module}/helm/azureIdentities.yaml.tpl") + template = file("${path.module}/files/azureIdentities.yaml.tpl") vars = { resourceID = azurerm_user_assigned_identity.velero[0].id clientID = azurerm_user_assigned_identity.velero[0].client_id From 7923ed8b7a969f9a7742f31abdab4b3094608c1c Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 24 Oct 2020 17:19:05 +0100 Subject: [PATCH 60/71] Added akv2k8s exception yaml --- terraform/files/akv2k8s-exception.yaml | 9 +++++++++ terraform/helm_akv2k8s.tf | 6 ++++-- terraform/variables.tf | 5 +++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 terraform/files/akv2k8s-exception.yaml diff --git a/terraform/files/akv2k8s-exception.yaml b/terraform/files/akv2k8s-exception.yaml new file mode 100644 index 0000000..b5dee1c --- /dev/null +++ b/terraform/files/akv2k8s-exception.yaml @@ -0,0 +1,9 @@ +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzurePodIdentityException +metadata: + name: akv2k8s-exception + namespace: akv2k8s # change if akv2k8s is installed in a different ns +spec: + podLabels: + app.kubernetes.io/instance: akv2k8s + app.kubernetes.io/name: controller diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 2e323a5..2e82218 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -41,8 +41,9 @@ resource "local_file" "kubeconfig" { resource "null_resource" "akv2k8s_crds" { triggers = { # always_run = "${timestamp()}" - akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) - cert_sync_yaml_contents = filemd5(var.cert_sync_yaml_path) + akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) + akv2k8s_exception_yaml_contents = filemd5(var.akv2k8s_exception_yaml_path) + cert_sync_yaml_contents = filemd5(var.cert_sync_yaml_path) } provisioner "local-exec" { @@ -52,6 +53,7 @@ resource "null_resource" "akv2k8s_crds" { # TODO: is this needed, or do CRDs install by default? # https://helm.sh/docs/chart_best_practices/custom_resource_definitions/ # kubectl apply -f ${var.akv2k8s_yaml_path} + kubectl apply -f ${var.akv2k8s_exception_yaml_path} kubectl apply -f ${var.cert_sync_yaml_path} EOT } diff --git a/terraform/variables.tf b/terraform/variables.tf index 0b0805f..d427530 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -288,10 +288,15 @@ variable "nexus_tls_secret_name" { # akv2k8s +# TODO: is this CRD file required? variable "akv2k8s_yaml_path" { default = "files/AzureKeyVaultSecret.yaml" } +variable "akv2k8s_exception_yaml_path" { + default = "files/akv2k8s-exception.yaml" +} + variable "cert_sync_yaml_path" { default = "files/akvs-certificate-sync.yaml" } From 53a4d74a20053a1d98ebfa7b5e80b60e35407982 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 24 Oct 2020 17:39:46 +0100 Subject: [PATCH 61/71] Updated velero helm values --- terraform/helm/velero_values.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/terraform/helm/velero_values.yaml b/terraform/helm/velero_values.yaml index 581bd1d..5f7a0e5 100644 --- a/terraform/helm/velero_values.yaml +++ b/terraform/helm/velero_values.yaml @@ -1,4 +1,5 @@ -# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.13.3/charts/velero/values.yaml +# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.13.6/charts/velero/values.yaml + ## ## Configuration settings that directly affect the Velero deployment YAML. ## @@ -8,7 +9,7 @@ image: # https://hub.docker.com/r/velero/velero/tags repository: velero/velero - tag: v1.5.1 + tag: v1.5.2 # Digest value example: sha256:d238835e151cec91c6a811fe3a89a66d3231d9f64d09e5f3c49552672d271f38. If used, it will # take precedence over the image.tag. # digest: @@ -21,8 +22,8 @@ image: # # If using kube2iam or kiam, use the following annotation with your AWS_ACCOUNT_ID # and VELERO_ROLE_NAME filled in: -# iam.amazonaws.com/role: arn:aws:iam:::role/ podAnnotations: {} + # iam.amazonaws.com/role: "arn:aws:iam:::role/" # Additional pod labels for Velero deployment's template. Optional # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ @@ -48,6 +49,9 @@ initContainers: securityContext: {} # fsGroup: 1337 +# Pod priority class name to use for the Velero deployment. Optional. +priorityClassName: "" + # Tolerations to use for the Velero deployment. Optional. tolerations: [] @@ -220,7 +224,7 @@ restic: podVolumePath: /var/lib/kubelet/pods privileged: false # Pod priority class name to use for the Restic daemonset. Optional. - priorityClassName: {} + priorityClassName: "" # Resource requests/limits to specify for the Restic daemonset deployment. Optional. resources: {} # Tolerations to use for the Restic daemonset. Optional. From 810b430d8f00c4d7172f56502b71b05739b78e61 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 24 Oct 2020 18:25:43 +0100 Subject: [PATCH 62/71] Added separate akv2k8s_exceptions null resource --- terraform/helm_akv2k8s.tf | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 2e82218..b306c08 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -42,7 +42,6 @@ resource "null_resource" "akv2k8s_crds" { triggers = { # always_run = "${timestamp()}" akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) - akv2k8s_exception_yaml_contents = filemd5(var.akv2k8s_exception_yaml_path) cert_sync_yaml_contents = filemd5(var.cert_sync_yaml_path) } @@ -53,7 +52,6 @@ resource "null_resource" "akv2k8s_crds" { # TODO: is this needed, or do CRDs install by default? # https://helm.sh/docs/chart_best_practices/custom_resource_definitions/ # kubectl apply -f ${var.akv2k8s_yaml_path} - kubectl apply -f ${var.akv2k8s_exception_yaml_path} kubectl apply -f ${var.cert_sync_yaml_path} EOT } @@ -73,6 +71,24 @@ resource "kubernetes_namespace" "akv2k8s" { depends_on = [null_resource.akv2k8s_crds] } +# https://www.terraform.io/docs/provisioners/local-exec.html +resource "null_resource" "akv2k8s_exceptions" { + triggers = { + # always_run = "${timestamp()}" + akv2k8s_exception_yaml_contents = filemd5(var.akv2k8s_exception_yaml_path) + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = < Date: Sat, 24 Oct 2020 18:43:09 +0100 Subject: [PATCH 63/71] Added akv2k8s_yaml_path for AzureKeyVaultSecret CRD --- terraform/helm_akv2k8s.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index b306c08..96dbf36 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -49,9 +49,8 @@ resource "null_resource" "akv2k8s_crds" { interpreter = ["/bin/bash", "-c"] command = < Date: Sat, 24 Oct 2020 18:45:30 +0100 Subject: [PATCH 64/71] Changed source RG resource --- terraform/helm_aad_pod_identity.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index 218922b..bc5a043 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -23,7 +23,7 @@ resource "azurerm_role_assignment" "aks_mi_velero_rg_mi_operator" { count = var.velero_enabled ? 1 : 0 principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id role_definition_name = "Managed Identity Operator" - scope = azurerm_user_assigned_identity.velero[0].resource_group_name + scope = azurerm_resource_group.velero[0].name skip_service_principal_aad_check = true } From 199a98814de838dde0594794e28e666f29fc64ba Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 24 Oct 2020 19:48:01 +0100 Subject: [PATCH 65/71] Update helm_aad_pod_identity.tf --- terraform/helm_aad_pod_identity.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index bc5a043..55c77f7 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -23,7 +23,7 @@ resource "azurerm_role_assignment" "aks_mi_velero_rg_mi_operator" { count = var.velero_enabled ? 1 : 0 principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id role_definition_name = "Managed Identity Operator" - scope = azurerm_resource_group.velero[0].name + scope = azurerm_resource_group.velero[0].id skip_service_principal_aad_check = true } From 6cc5cfde9408ec52cbf9964cbee2f15c5ad82964 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 25 Oct 2020 06:43:31 +0000 Subject: [PATCH 66/71] Bumped versions --- terraform/providers.tf | 2 +- terraform/variables.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/providers.tf b/terraform/providers.tf index 376a756..7ab3e16 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -30,7 +30,7 @@ terraform { # must include blank features block # https://github.com/terraform-providers/terraform-provider-azurerm/releases provider "azurerm" { - version = "2.32.0" + version = "2.33.0" features {} } diff --git a/terraform/variables.tf b/terraform/variables.tf index d427530..0458e37 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -17,7 +17,7 @@ variable "kubernetes_version" { # https://github.com/kubernetes/ingress-nginx/releases # https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/Chart.yaml#L3 variable "nginx_chart_version" { - default = "3.4.0" + default = "3.7.1" } # https://hub.helm.sh/charts/jetstack/cert-manager @@ -44,7 +44,7 @@ variable "nexus_chart_version" { # https://github.com/SparebankenVest/helm-charts/tree/gh-pages/akv2k8s # https://github.com/SparebankenVest/public-helm-charts/blob/master/stable/akv2k8s/Chart.yaml#L5 variable "akv2k8s_chart_version" { - default = "1.1.25" + default = "1.1.26" } # https://github.com/Azure/aad-pod-identity/blob/master/charts/aad-pod-identity/Chart.yaml#L4 From 54addfdd2c540a531470789d465b5bf230ea313c Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 25 Oct 2020 07:34:30 +0000 Subject: [PATCH 67/71] Revert versions --- terraform/providers.tf | 2 +- terraform/variables.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/providers.tf b/terraform/providers.tf index 7ab3e16..376a756 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -30,7 +30,7 @@ terraform { # must include blank features block # https://github.com/terraform-providers/terraform-provider-azurerm/releases provider "azurerm" { - version = "2.33.0" + version = "2.32.0" features {} } diff --git a/terraform/variables.tf b/terraform/variables.tf index 0458e37..d427530 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -17,7 +17,7 @@ variable "kubernetes_version" { # https://github.com/kubernetes/ingress-nginx/releases # https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/Chart.yaml#L3 variable "nginx_chart_version" { - default = "3.7.1" + default = "3.4.0" } # https://hub.helm.sh/charts/jetstack/cert-manager @@ -44,7 +44,7 @@ variable "nexus_chart_version" { # https://github.com/SparebankenVest/helm-charts/tree/gh-pages/akv2k8s # https://github.com/SparebankenVest/public-helm-charts/blob/master/stable/akv2k8s/Chart.yaml#L5 variable "akv2k8s_chart_version" { - default = "1.1.26" + default = "1.1.25" } # https://github.com/Azure/aad-pod-identity/blob/master/charts/aad-pod-identity/Chart.yaml#L4 From 8953ab648d3d80bbf9da942b45097df33e92eb42 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 25 Oct 2020 07:36:42 +0000 Subject: [PATCH 68/71] Changed velero to only backup nexus namespace --- terraform/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index d427530..f69b3ec 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -200,7 +200,7 @@ variable "velero_backup_schedule" { variable "velero_backup_included_namespaces" { type = list(string) default = [ - "*" + "nexus" ] } From 41f69ea503122bc9e1719854654e9240b13e2a00 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 25 Oct 2020 08:39:48 +0000 Subject: [PATCH 69/71] Removed old velero SP auth --- terraform/velero_auth.tf | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 terraform/velero_auth.tf diff --git a/terraform/velero_auth.tf b/terraform/velero_auth.tf deleted file mode 100644 index da621b8..0000000 --- a/terraform/velero_auth.tf +++ /dev/null @@ -1,37 +0,0 @@ -# TODO remove this after confirming MI auth works -# Service Principle for velero -# resource "azuread_application" "velero_sp" { -# name = var.velero_service_principle_name -# } - -# resource "azuread_service_principal" "velero_sp" { -# application_id = azuread_application.velero_sp.application_id -# } - -# resource "random_string" "velero_sp" { -# length = 16 -# special = true -# keepers = { -# service_principal = azuread_service_principal.velero_sp.id -# } -# } - -# resource "azuread_service_principal_password" "velero_sp" { -# service_principal_id = azuread_service_principal.velero_sp.id -# value = random_string.velero_sp.result -# end_date_relative = "8760h" # 8760h = 1 year - -# lifecycle { -# ignore_changes = [end_date] -# } -# } - - -# # Service Principle role assignments -# resource "azurerm_role_assignment" "velero_sp_to_sub" { -# principal_id = azuread_service_principal.velero_sp.id -# role_definition_name = "Contributor" -# scope = data.azurerm_subscription.current.id -# skip_service_principal_aad_check = true -# depends_on = [azuread_service_principal_password.velero_sp] -# } From 0c348dae6b788156194c4a10cf3e6ddc3cbb8b07 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 25 Oct 2020 08:40:01 +0000 Subject: [PATCH 70/71] Added velero_mi_aks_node_rg_vm_contributor --- terraform/velero_mi_auth.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/terraform/velero_mi_auth.tf b/terraform/velero_mi_auth.tf index b5174d0..db05690 100644 --- a/terraform/velero_mi_auth.tf +++ b/terraform/velero_mi_auth.tf @@ -14,3 +14,13 @@ resource "azurerm_role_assignment" "velero_mi_velero_storage_rg" { role_definition_name = "Contributor" scope = azurerm_resource_group.velero[0].id } + +# https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles?source=docs#virtual-machine-contributor +# vm disk read and write action / perms +resource "azurerm_role_assignment" "velero_mi_aks_node_rg_vm_contributor" { + count = var.velero_enabled ? 1 : 0 + principal_id = azurerm_user_assigned_identity.velero[0].principal_id + role_definition_name = "Virtual Machine Contributor" + scope = data.azurerm_resource_group.aks.id + skip_service_principal_aad_check = true +} From f0842c44c36446f3aab634a17691d5760366da27 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 25 Oct 2020 09:15:01 +0000 Subject: [PATCH 71/71] Changed data resource name for aks_node_rg --- terraform/data.tf | 4 ++++ terraform/function_app.tf | 6 +----- terraform/helm_aad_pod_identity.tf | 4 ++-- terraform/velero_mi_auth.tf | 7 +++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/terraform/data.tf b/terraform/data.tf index 56510ac..6e06f9a 100644 --- a/terraform/data.tf +++ b/terraform/data.tf @@ -4,3 +4,7 @@ data "azurerm_subscription" "current" {} data "azuread_group" "aks" { name = var.aad_group_name } + +data "azurerm_resource_group" "aks_node_rg" { + name = azurerm_kubernetes_cluster.aks.node_resource_group +} diff --git a/terraform/function_app.tf b/terraform/function_app.tf index 3145930..e2d625f 100644 --- a/terraform/function_app.tf +++ b/terraform/function_app.tf @@ -120,12 +120,8 @@ resource "azurerm_function_app" "func_app" { # Give Function App Reader role for the AKS cluster node resource group -data "azurerm_resource_group" "aks" { - name = azurerm_kubernetes_cluster.aks.node_resource_group -} - resource "azurerm_role_assignment" "func_app" { - scope = data.azurerm_resource_group.aks.id + scope = data.azurerm_resource_group.aks_node_rg.id role_definition_name = "Reader" principal_id = azurerm_function_app.func_app.identity.0.principal_id } diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/helm_aad_pod_identity.tf index 55c77f7..d36e336 100644 --- a/terraform/helm_aad_pod_identity.tf +++ b/terraform/helm_aad_pod_identity.tf @@ -6,7 +6,7 @@ resource "azurerm_role_assignment" "aks_mi_aks_node_rg_vm_contributor" { count = var.velero_enabled ? 1 : 0 principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id role_definition_name = "Virtual Machine Contributor" - scope = data.azurerm_resource_group.aks.id + scope = data.azurerm_resource_group.aks_node_rg.id skip_service_principal_aad_check = true } @@ -14,7 +14,7 @@ resource "azurerm_role_assignment" "aks_mi_aks_node_rg_mi_operator" { count = var.velero_enabled ? 1 : 0 principal_id = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id role_definition_name = "Managed Identity Operator" - scope = data.azurerm_resource_group.aks.id + scope = data.azurerm_resource_group.aks_node_rg.id skip_service_principal_aad_check = true } diff --git a/terraform/velero_mi_auth.tf b/terraform/velero_mi_auth.tf index db05690..52a99e7 100644 --- a/terraform/velero_mi_auth.tf +++ b/terraform/velero_mi_auth.tf @@ -15,12 +15,11 @@ resource "azurerm_role_assignment" "velero_mi_velero_storage_rg" { scope = azurerm_resource_group.velero[0].id } -# https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles?source=docs#virtual-machine-contributor -# vm disk read and write action / perms +# assign velero MI contributor rights to velero storage RG resource "azurerm_role_assignment" "velero_mi_aks_node_rg_vm_contributor" { count = var.velero_enabled ? 1 : 0 principal_id = azurerm_user_assigned_identity.velero[0].principal_id - role_definition_name = "Virtual Machine Contributor" - scope = data.azurerm_resource_group.aks.id + role_definition_name = "Contributor" + scope = data.azurerm_resource_group.aks_node_rg.id skip_service_principal_aad_check = true }