From f49e26208c91654fa1bb5f9ecda73991da77e299 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 18 Mar 2021 17:59:47 +0000 Subject: [PATCH 01/58] Added admin user to argo rbac --- terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml b/terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml index 2a55d75d..33931260 100644 --- a/terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml +++ b/terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml @@ -13,4 +13,5 @@ data: # (Nested groups work fine) # g, , role:admin policy.csv: | + g, admin, role:admin g, $ARGO_ADMIN_GROUP_ID, role:admin From 44ebfb1f0a9f617c6f0f2ff4ca18f3f90ef169a0 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 18 Mar 2021 18:15:49 +0000 Subject: [PATCH 02/58] Changed terraform config to v0.13.6 --- .devcontainer/Dockerfile | 2 +- .devcontainer/devcontainer.json | 2 +- .github/workflows/build.yml | 2 +- .github/workflows/destroy.yml | 2 +- terraform/providers.tf | 64 +++++++++++++++++++++++++-------- 5 files changed, 54 insertions(+), 18 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 15500146..a06abd8f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -45,7 +45,7 @@ RUN if [ "${INSTALL_AZURE_CLI}" = "true" ]; then bash /tmp/library-scripts/azcli # Install Terraform, tflint, Go, PowerShell, and other useful tools # TODO: move this into main "RUN" layer above -ARG TERRAFORM_VERSION=0.12.30 +ARG TERRAFORM_VERSION=0.13.6 ARG TFLINT_VERSION=0.18.0 RUN bash /tmp/library-scripts/terraform-debian.sh "${TERRAFORM_VERSION}" "${TFLINT_VERSION}" \ && bash /tmp/library-scripts/powershell-debian.sh \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 40a933d3..bf002d44 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "TERRAFORM_VERSION": "0.12.30", + "TERRAFORM_VERSION": "0.13.6", "TFLINT_VERSION": "0.22.0", "INSTALL_AZURE_CLI": "true", "INSTALL_DOCKER": "true", diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d6f06a5f..59d39e2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,7 +57,7 @@ env: TF_IN_AUTOMATION: "true" TF_INPUT: "false" TF_PLAN: "tfplan" - TF_VERSION: "0.12.30" # "latest" is supported + TF_VERSION: "0.13.6" # "latest" is supported TF_WORKING_DIR: ./terraform # https://github.com/terraform-linters/tflint-ruleset-azurerm/releases TFLINT_RULESET_AZURERM_VERSION: "v0.8.2" diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 20186934..024940fe 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -49,7 +49,7 @@ env: # terraform TF_IN_AUTOMATION: "true" TF_INPUT: "false" - TF_VERSION: "0.12.30" # "latest" is supported + TF_VERSION: "0.13.6" # "latest" is supported TF_WORKING_DIR: terraform # Env var concatenation is currently not supported at Workflow or Job scope. See workaround below: diff --git a/terraform/providers.tf b/terraform/providers.tf index af623f3e..5a57b83b 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -1,4 +1,9 @@ terraform { + + # https://github.com/hashicorp/terraform/releases + # 0.13.X + required_version = "~> 0.13.6" + # terraform remote state backend "azurerm" { access_key = "__STORAGE_KEY__" @@ -8,34 +13,64 @@ terraform { } # providers (pin all versions) - # versioning syntax: https://www.terraform.io/docs/configuration/modules.html#module-versions + # versioning syntax: https://www.terraform.io/docs/language/expressions/version-constraints.html#version-constraint-syntax required_providers { - # https://github.com/hashicorp/terraform-provider-helm/releases - helm = "2.0.3" - # https://github.com/hashicorp/terraform-provider-kubernetes/releases - kubernetes = "2.0.2" + # https://github.com/terraform-providers/terraform-provider-azurerm/releases + azurerm = { + source = "hashicorp/azurerm" + version = "2.51.0" + } # https://github.com/terraform-providers/terraform-provider-azuread/releases - azuread = "1.4.0" + azuread = { + source = "hashicorp/azuread" + version = "1.4.0" + } - random = "~> 2.2" # ~> 2.2 = 2.X.Y - tls = "~> 2.1" - } + # https://github.com/hashicorp/terraform-provider-kubernetes/releases + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.0.3" + } - # 0.12.X - required_version = "~> 0.12.30" # https://github.com/hashicorp/terraform/releases + # https://github.com/hashicorp/terraform-provider-helm/releases + helm = { + source = "hashicorp/helm" + version = "2.0.3" + } + + random = { + source = "hashicorp/random" + version = "~> 2.2" + } + + tls = { + source = "hashicorp/tls" + version = "~> 2.1" + } + + local = { + source = "hashicorp/local" + } + + null = { + source = "hashicorp/null" + } + + template = { + source = "hashicorp/template" + } + } } # must include blank features block # https://github.com/terraform-providers/terraform-provider-azurerm/releases provider "azurerm" { - version = "2.51.0" features {} } -# use statically defined credentials -# https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#statically-defined-credentials +# https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs#credentials-config provider "kubernetes" { host = module.aks.full_object.kube_admin_config[0].host client_certificate = base64decode(module.aks.full_object.kube_admin_config[0].client_certificate) @@ -43,6 +78,7 @@ provider "kubernetes" { cluster_ca_certificate = base64decode(module.aks.full_object.kube_admin_config[0].cluster_ca_certificate) } +# https://registry.terraform.io/providers/hashicorp/helm/latest/docs#credentials-config provider "helm" { kubernetes { host = module.aks.full_object.kube_admin_config[0].host From 4e0da719c90c99336fe3b782e85422af0c190d31 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 18 Mar 2021 18:23:02 +0000 Subject: [PATCH 03/58] Added version constraints --- terraform/providers.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/providers.tf b/terraform/providers.tf index 5a57b83b..1820248a 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -52,14 +52,17 @@ terraform { local = { source = "hashicorp/local" + version = "~> 2.0" } null = { source = "hashicorp/null" + version = "~> 3.0" } template = { source = "hashicorp/template" + version = "~> 2.0" } } } From 137925b4a9e08050dbcb16c9380de7760407e3c2 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 18 Mar 2021 18:23:46 +0000 Subject: [PATCH 04/58] Fixed terraform Interpolation-only expression warnings --- terraform/helm_argocd.tf | 2 +- terraform/helm_kured.tf | 2 +- terraform/helm_nexus.tf | 2 +- terraform/helm_nginx.tf | 2 +- terraform/velero.tf | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/helm_argocd.tf b/terraform/helm_argocd.tf index 744a2d71..6995a452 100644 --- a/terraform/helm_argocd.tf +++ b/terraform/helm_argocd.tf @@ -45,7 +45,7 @@ resource "helm_release" "argocd" { version = var.argocd_chart_version timeout = 600 atomic = true - values = ["${file("${path.module}/files/argocd-values.yaml")}"] + values = [file("${path.module}/files/argocd-values.yaml")] set { name = "global.image.tag" diff --git a/terraform/helm_kured.tf b/terraform/helm_kured.tf index 1ce3b271..f8ebfbba 100644 --- a/terraform/helm_kured.tf +++ b/terraform/helm_kured.tf @@ -24,7 +24,7 @@ resource "helm_release" "kured" { timeout = 600 atomic = true - values = ["${file("helm/kured_values.yaml")}"] + values = [file("helm/kured_values.yaml")] set { name = "image.tag" diff --git a/terraform/helm_nexus.tf b/terraform/helm_nexus.tf index 958ab924..0fdd5695 100644 --- a/terraform/helm_nexus.tf +++ b/terraform/helm_nexus.tf @@ -22,7 +22,7 @@ resource "helm_release" "nexus" { timeout = 600 atomic = true - values = ["${file("helm/nexus_values.yaml")}"] + values = [file("helm/nexus_values.yaml")] set { name = "image.tag" diff --git a/terraform/helm_nginx.tf b/terraform/helm_nginx.tf index 3a4ac0ad..391464e2 100644 --- a/terraform/helm_nginx.tf +++ b/terraform/helm_nginx.tf @@ -21,7 +21,7 @@ resource "helm_release" "nginx" { version = var.nginx_chart_version timeout = 600 atomic = true - values = ["${file("helm/nginx_values.yaml")}"] + values = [file("helm/nginx_values.yaml")] set { name = "controller.admissionWebhooks.enabled" diff --git a/terraform/velero.tf b/terraform/velero.tf index 9e9b5540..2b1318d8 100644 --- a/terraform/velero.tf +++ b/terraform/velero.tf @@ -88,7 +88,7 @@ resource "helm_release" "velero" { timeout = 600 atomic = true - values = ["${file("helm/velero_values.yaml")}"] + values = [file("helm/velero_values.yaml")] set { name = "configuration.backupStorageLocation.config.resourceGroup" From d58c4686df6534cf8b6bc481aac4efd0d4c5abc3 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 18 Mar 2021 19:02:39 +0000 Subject: [PATCH 05/58] Renamed providers.tf to versions.tf --- terraform/{providers.tf => versions.tf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename terraform/{providers.tf => versions.tf} (100%) diff --git a/terraform/providers.tf b/terraform/versions.tf similarity index 100% rename from terraform/providers.tf rename to terraform/versions.tf From 508d041ddf7d4d12ee5166362424be234a326cfa Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 19 Mar 2021 06:56:46 +0000 Subject: [PATCH 06/58] Bumped provider / app versions --- terraform/variables.tf | 20 ++++++++++---------- terraform/versions.tf | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index d845623d..2ed2d2a7 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -6,7 +6,7 @@ # https://github.com/Azure/AKS/releases # az aks get-versions --location uksouth --output table variable "kubernetes_version" { - default = "1.17.16" + default = "1.18.14" } # Helm charts @@ -21,7 +21,7 @@ variable "kubernetes_version" { # helm search repo ingress-nginx/ingress-nginx # * also update terraform/helm/nginx_values.yaml variable "nginx_chart_version" { - default = "3.23.0" + default = "3.24.0" } # https://hub.helm.sh/charts/jetstack/cert-manager @@ -34,12 +34,12 @@ variable "cert_manager_chart_version" { # helm search repo vmware-tanzu/velero # * also update terraform/helm/velero_values.yaml variable "velero_chart_version" { - default = "2.14.8" + default = "2.15.0" } # https://hub.docker.com/r/sonatype/nexus3/tags variable "nexus_image_tag" { - default = "3.29.2" + default = "3.30.0" } # https://github.com/adamrushuk/charts/releases @@ -54,7 +54,7 @@ variable "nexus_chart_version" { # https://github.com/SparebankenVest/public-helm-charts/blob/master/stable/akv2k8s/Chart.yaml#L5 # helm search repo spv-charts/akv2k8s variable "akv2k8s_chart_version" { - default = "1.1.28" + default = "2.0.7" } # https://github.com/Azure/aad-pod-identity/blob/master/charts/aad-pod-identity/Chart.yaml#L4 @@ -67,18 +67,18 @@ variable "aad_pod_identity_chart_version" { # https://github.com/bitnami/charts/blob/master/bitnami/external-dns/Chart.yaml#L21 # helm search repo bitnami/external-dns variable "external_dns_chart_version" { - default = "4.8.0" + default = "4.9.1" } # https://github.com/weaveworks/kured/tree/master/charts/kured # helm search repo kured/kured variable "kured_chart_version" { - default = "2.3.2" + default = "2.4.0" } # https://github.com/weaveworks/kured#kubernetes--os-compatibility variable "kured_image_tag" { - default = "1.5.1" + default = "1.6.0" } @@ -86,12 +86,12 @@ variable "kured_image_tag" { # https://github.com/argoproj/argo-helm/blob/master/charts/argo-cd/Chart.yaml#L5 # helm search repo argo/argo-cd variable "argocd_chart_version" { - default = "2.14.6" + default = "2.17.1" } # https://hub.docker.com/r/argoproj/argocd/tags variable "argocd_image_tag" { - default = "v1.8.4" + default = "v1.8.7" } #endregion Versions diff --git a/terraform/versions.tf b/terraform/versions.tf index 1820248a..ce28eaf2 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -19,25 +19,25 @@ terraform { # https://github.com/terraform-providers/terraform-provider-azurerm/releases azurerm = { source = "hashicorp/azurerm" - version = "2.51.0" + version = "~> 2.52.0" } # https://github.com/terraform-providers/terraform-provider-azuread/releases azuread = { source = "hashicorp/azuread" - version = "1.4.0" + version = "~> 1.4.0" } # https://github.com/hashicorp/terraform-provider-kubernetes/releases kubernetes = { source = "hashicorp/kubernetes" - version = "2.0.3" + version = "~> 2.0.3" } # https://github.com/hashicorp/terraform-provider-helm/releases helm = { source = "hashicorp/helm" - version = "2.0.3" + version = "~> 2.0.3" } random = { From c06a1cc706ca0f5a00022cf9c03b55e588892dec Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 19 Mar 2021 07:28:36 +0000 Subject: [PATCH 07/58] Change kured to v1.6.1 --- terraform/variables.tf | 2 +- terraform/versions.tf | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 2ed2d2a7..04f01e68 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -78,7 +78,7 @@ variable "kured_chart_version" { # https://github.com/weaveworks/kured#kubernetes--os-compatibility variable "kured_image_tag" { - default = "1.6.0" + default = "1.6.1" } diff --git a/terraform/versions.tf b/terraform/versions.tf index ce28eaf2..9e6036e2 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -1,6 +1,7 @@ terraform { # https://github.com/hashicorp/terraform/releases + # https://github.com/hashicorp/terraform/blob/main/CHANGELOG.md # 0.13.X required_version = "~> 0.13.6" From 16c3a465c6535c0732e6a96851f27137a968ffe8 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 19 Mar 2021 08:06:42 +0000 Subject: [PATCH 08/58] Changed akvs config for v2 helm chart --- terraform/files/akvs-certificate-sync.yaml | 2 +- .../files/argocd-akvs-certificate-sync.yaml | 2 +- .../files/gitlab-akvs-certificate-sync.yaml | 2 +- terraform/helm_akv2k8s.tf | 81 ++++++++++--------- terraform/variables.tf | 12 +-- 5 files changed, 52 insertions(+), 47 deletions(-) diff --git a/terraform/files/akvs-certificate-sync.yaml b/terraform/files/akvs-certificate-sync.yaml index 0e93d816..f94a13ed 100644 --- a/terraform/files/akvs-certificate-sync.yaml +++ b/terraform/files/akvs-certificate-sync.yaml @@ -1,5 +1,5 @@ # https://akv2k8s.io/tutorials/sync/2-certificate/ -apiVersion: spv.no/v1alpha1 +apiVersion: spv.no/v2beta1 kind: AzureKeyVaultSecret metadata: name: certificate-sync diff --git a/terraform/files/argocd-akvs-certificate-sync.yaml b/terraform/files/argocd-akvs-certificate-sync.yaml index 9d1f2d84..120a53d8 100644 --- a/terraform/files/argocd-akvs-certificate-sync.yaml +++ b/terraform/files/argocd-akvs-certificate-sync.yaml @@ -1,5 +1,5 @@ # https://akv2k8s.io/tutorials/sync/2-certificate/ -apiVersion: spv.no/v1alpha1 +apiVersion: spv.no/v2beta1 kind: AzureKeyVaultSecret metadata: name: argocd-certificate-sync diff --git a/terraform/files/gitlab-akvs-certificate-sync.yaml b/terraform/files/gitlab-akvs-certificate-sync.yaml index 0dbfcc06..73ffcad3 100644 --- a/terraform/files/gitlab-akvs-certificate-sync.yaml +++ b/terraform/files/gitlab-akvs-certificate-sync.yaml @@ -1,5 +1,5 @@ # https://akv2k8s.io/tutorials/sync/2-certificate/ -apiVersion: spv.no/v1alpha1 +apiVersion: spv.no/v2beta1 kind: AzureKeyVaultSecret metadata: name: gitlab-certificate-sync diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 7e615087..4bd2f80b 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -40,23 +40,23 @@ resource "local_file" "kubeconfig" { } # https://www.terraform.io/docs/provisioners/local-exec.html -resource "null_resource" "akv2k8s_crds" { - triggers = { - # always_run = "${timestamp()}" - akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) - } - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - command = < Date: Fri, 19 Mar 2021 09:17:53 +0000 Subject: [PATCH 09/58] Removed akv2k8s exception and CRD --- terraform/files/AzureKeyVaultSecret.yaml | 103 ----------------------- terraform/files/akv2k8s-exception.yaml | 9 -- terraform/helm_akv2k8s.tf | 62 -------------- terraform/variables.tf | 12 +-- 4 files changed, 2 insertions(+), 184 deletions(-) delete mode 100644 terraform/files/AzureKeyVaultSecret.yaml delete mode 100644 terraform/files/akv2k8s-exception.yaml diff --git a/terraform/files/AzureKeyVaultSecret.yaml b/terraform/files/AzureKeyVaultSecret.yaml deleted file mode 100644 index b3f5b406..00000000 --- a/terraform/files/AzureKeyVaultSecret.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# https://raw.githubusercontent.com/sparebankenvest/azure-key-vault-to-kubernetes/crd-1.1.0/crds/AzureKeyVaultSecret.yaml -# -# https://akv2k8s.io/reference/azure-key-vault-secret/ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: azurekeyvaultsecrets.spv.no - labels: - app.kubernetes.io/name: akv2k8s - annotations: - "helm.sh/resource-policy": keep -spec: - group: spv.no - names: - kind: AzureKeyVaultSecret - listKind: AzureKeyVaultSecretList - plural: azurekeyvaultsecrets - singular: azurekeyvaultsecret - shortNames: - - akvs - categories: - - all - additionalPrinterColumns: - - name: Vault - type: string - description: Which Azure Key Vault this resource is asosiated with - JSONPath: .spec.vault.name - - name: Vault Object - type: string - description: Which Azure Key Vault object this resource is asosiated with - JSONPath: .spec.vault.object.name - - name: Secret Name - type: string - description: Which Kubernetes Secret this resource is synched with, if any - JSONPath: .status.secretName - - name: Synched - type: string - description: When this resource was last synched with Azure Key Vault - JSONPath: .status.lastAzureUpdate - scope: Namespaced - versions: - - name: v1alpha1 - served: true - storage: false - - name: v1 - served: true - storage: true - - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - required: ['vault'] - properties: - vault: - required: ['name', 'object'] - properties: - name: - type: string - description: Name of the Azure Key Vault - object: - required: ['name', 'type'] - properties: - name: - type: string - description: The object name in Azure Key Vault - type: - type: string - description: The type of object in Azure Key Vault - enum: - - secret - - certificate - - key - - multi-key-value-secret - version: - type: string - description: The object version in Azure Key Vault - contentType: - type: string - description: Only used when type is multi-key-value-secret. - enum: - - application/x-json - - application/x-yaml - output: - properties: - transform: - type: array - items: - type: string - secret: - required: ['name'] - properties: - name: - type: string - description: Name for Kubernetes secret - type: - type: string - description: Type of Secret in Kubernetes - dataKey: - type: string - description: The key to use in Kubernetes secret when setting the value from Azure Keyv Vault object data diff --git a/terraform/files/akv2k8s-exception.yaml b/terraform/files/akv2k8s-exception.yaml deleted file mode 100644 index b5dee1c0..00000000 --- a/terraform/files/akv2k8s-exception.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: "aadpodidentity.k8s.io/v1" -kind: AzurePodIdentityException -metadata: - name: akv2k8s-exception - namespace: akv2k8s # change if akv2k8s is installed in a different ns -spec: - podLabels: - app.kubernetes.io/instance: akv2k8s - app.kubernetes.io/name: controller diff --git a/terraform/helm_akv2k8s.tf b/terraform/helm_akv2k8s.tf index 4bd2f80b..df7e133d 100644 --- a/terraform/helm_akv2k8s.tf +++ b/terraform/helm_akv2k8s.tf @@ -39,25 +39,6 @@ resource "local_file" "kubeconfig" { depends_on = [module.aks] } -# https://www.terraform.io/docs/provisioners/local-exec.html -# resource "null_resource" "akv2k8s_crds" { -# triggers = { -# # always_run = "${timestamp()}" -# akv2k8s_yaml_contents = filemd5(var.akv2k8s_yaml_path) -# } - -# provisioner "local-exec" { -# interpreter = ["/bin/bash", "-c"] -# command = < Date: Fri, 19 Mar 2021 09:18:22 +0000 Subject: [PATCH 10/58] Changed nexus_cert_sync resource location --- terraform/helm_nexus.tf | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/terraform/helm_nexus.tf b/terraform/helm_nexus.tf index 0fdd5695..d5ac39ce 100644 --- a/terraform/helm_nexus.tf +++ b/terraform/helm_nexus.tf @@ -12,6 +12,28 @@ resource "kubernetes_namespace" "nexus" { depends_on = [module.aks] } +# https://www.terraform.io/docs/provisioners/local-exec.html +resource "null_resource" "nexus_cert_sync" { + triggers = { + # always_run = "${timestamp()}" + cert_sync_yaml_contents = filemd5(var.nexus_cert_sync_yaml_path) + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = < Date: Fri, 19 Mar 2021 10:33:30 +0000 Subject: [PATCH 11/58] Changed filename for nexus-akvs-certificate-sync.yaml --- ...kvs-certificate-sync.yaml => nexus-akvs-certificate-sync.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename terraform/files/{akvs-certificate-sync.yaml => nexus-akvs-certificate-sync.yaml} (100%) diff --git a/terraform/files/akvs-certificate-sync.yaml b/terraform/files/nexus-akvs-certificate-sync.yaml similarity index 100% rename from terraform/files/akvs-certificate-sync.yaml rename to terraform/files/nexus-akvs-certificate-sync.yaml From cbea5ba25b93a64af8e09fc353dfcb9467efe4be Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 20 Mar 2021 11:00:33 +0000 Subject: [PATCH 12/58] Added terraform logging --- .github/workflows/destroy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 024940fe..ef2993c5 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -49,6 +49,8 @@ env: # terraform TF_IN_AUTOMATION: "true" TF_INPUT: "false" + TF_LOG_PATH: terraform.log + TF_LOG: DEBUG TF_VERSION: "0.13.6" # "latest" is supported TF_WORKING_DIR: terraform From 01d7b1e0e6e57d91e10aedd59f96e36355b47eae Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 20 Mar 2021 11:03:37 +0000 Subject: [PATCH 13/58] Added terraform logging --- .github/workflows/destroy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index ef2993c5..553a1406 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -50,7 +50,7 @@ env: TF_IN_AUTOMATION: "true" TF_INPUT: "false" TF_LOG_PATH: terraform.log - TF_LOG: DEBUG + TF_LOG: TRACE TF_VERSION: "0.13.6" # "latest" is supported TF_WORKING_DIR: terraform From f07c89c9477b215935d02ef2c35018f10eb548e0 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 20 Mar 2021 11:07:41 +0000 Subject: [PATCH 14/58] Added tf log upload --- .github/workflows/destroy.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 553a1406..60576aae 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -123,6 +123,13 @@ jobs: terraform destroy -no-color -auto-approve working-directory: ${{ env.TF_WORKING_DIR }} + - name: Terraform logs + uses: actions/upload-artifact@v2 + with: + name: Terraform logs + path: ${{ env.TF_WORKING_DIR }}/${{ env.TF_LOG_PATH }} + if: always() + # Cleanup - name: Delete Storage run: ./scripts/storage_delete.sh From 45e27132bcc4319a022dfef5f9148f82f67193db Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 25 Mar 2021 07:58:43 +0000 Subject: [PATCH 15/58] Changed terraform config to v0.14.9 --- .devcontainer/Dockerfile | 2 +- .devcontainer/devcontainer.json | 2 +- .github/workflows/build.yml | 2 +- .github/workflows/destroy.yml | 2 +- terraform/versions.tf | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a06abd8f..f48e60d3 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -45,7 +45,7 @@ RUN if [ "${INSTALL_AZURE_CLI}" = "true" ]; then bash /tmp/library-scripts/azcli # Install Terraform, tflint, Go, PowerShell, and other useful tools # TODO: move this into main "RUN" layer above -ARG TERRAFORM_VERSION=0.13.6 +ARG TERRAFORM_VERSION=0.14.9 ARG TFLINT_VERSION=0.18.0 RUN bash /tmp/library-scripts/terraform-debian.sh "${TERRAFORM_VERSION}" "${TFLINT_VERSION}" \ && bash /tmp/library-scripts/powershell-debian.sh \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bf002d44..670c689d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "TERRAFORM_VERSION": "0.13.6", + "TERRAFORM_VERSION": "0.14.9", "TFLINT_VERSION": "0.22.0", "INSTALL_AZURE_CLI": "true", "INSTALL_DOCKER": "true", diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59d39e2f..b1bd8b3b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,7 +57,7 @@ env: TF_IN_AUTOMATION: "true" TF_INPUT: "false" TF_PLAN: "tfplan" - TF_VERSION: "0.13.6" # "latest" is supported + TF_VERSION: "0.14.9" # "latest" is supported TF_WORKING_DIR: ./terraform # https://github.com/terraform-linters/tflint-ruleset-azurerm/releases TFLINT_RULESET_AZURERM_VERSION: "v0.8.2" diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 60576aae..7ee337a5 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -51,7 +51,7 @@ env: TF_INPUT: "false" TF_LOG_PATH: terraform.log TF_LOG: TRACE - TF_VERSION: "0.13.6" # "latest" is supported + TF_VERSION: "0.14.9" # "latest" is supported TF_WORKING_DIR: terraform # Env var concatenation is currently not supported at Workflow or Job scope. See workaround below: diff --git a/terraform/versions.tf b/terraform/versions.tf index 9e6036e2..bb4936a2 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -2,8 +2,8 @@ terraform { # https://github.com/hashicorp/terraform/releases # https://github.com/hashicorp/terraform/blob/main/CHANGELOG.md - # 0.13.X - required_version = "~> 0.13.6" + # 0.14.X + required_version = "~> 0.14.9" # terraform remote state backend "azurerm" { From 7820b2536c8a2fd51209f0185c934bb4d2cc3317 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 25 Mar 2021 08:31:43 +0000 Subject: [PATCH 16/58] Changed min tf version to 0.13 --- terraform/versions.tf | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/terraform/versions.tf b/terraform/versions.tf index bb4936a2..309baac9 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -2,8 +2,7 @@ terraform { # https://github.com/hashicorp/terraform/releases # https://github.com/hashicorp/terraform/blob/main/CHANGELOG.md - # 0.14.X - required_version = "~> 0.14.9" + required_version = ">= 0.13" # terraform remote state backend "azurerm" { @@ -19,13 +18,13 @@ terraform { # https://github.com/terraform-providers/terraform-provider-azurerm/releases azurerm = { - source = "hashicorp/azurerm" + source = "hashicorp/azurerm" version = "~> 2.52.0" } # https://github.com/terraform-providers/terraform-provider-azuread/releases azuread = { - source = "hashicorp/azuread" + source = "hashicorp/azuread" version = "~> 1.4.0" } @@ -37,7 +36,7 @@ terraform { # https://github.com/hashicorp/terraform-provider-helm/releases helm = { - source = "hashicorp/helm" + source = "hashicorp/helm" version = "~> 2.0.3" } @@ -52,17 +51,17 @@ terraform { } local = { - source = "hashicorp/local" + source = "hashicorp/local" version = "~> 2.0" } null = { - source = "hashicorp/null" + source = "hashicorp/null" version = "~> 3.0" } template = { - source = "hashicorp/template" + source = "hashicorp/template" version = "~> 2.0" } } From 0e87db345b6bd30301e05607bd943e1924c4c52e Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 26 Mar 2021 07:12:59 +0000 Subject: [PATCH 17/58] Bumped azurerm provider to 2.53.0 --- terraform/versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/versions.tf b/terraform/versions.tf index 309baac9..52cf6a4d 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -19,7 +19,7 @@ terraform { # https://github.com/terraform-providers/terraform-provider-azurerm/releases azurerm = { source = "hashicorp/azurerm" - version = "~> 2.52.0" + version = "~> 2.53.0" } # https://github.com/terraform-providers/terraform-provider-azuread/releases From 19112a6c208244eb3e2b22f39bb7be5ad64a944b Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 26 Mar 2021 08:58:17 +0000 Subject: [PATCH 18/58] Bumped adamrushuk/aks/azurerm tf module to v0.8.0 --- terraform/aks.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/aks.tf b/terraform/aks.tf index 5fac5dba..9ec36ccf 100644 --- a/terraform/aks.tf +++ b/terraform/aks.tf @@ -54,7 +54,7 @@ resource "azurerm_log_analytics_solution" "aks" { # https://registry.terraform.io/modules/adamrushuk/aks/azurerm/latest module "aks" { source = "adamrushuk/aks/azurerm" - version = "0.7.0" + version = "0.8.0" kubernetes_version = var.kubernetes_version location = azurerm_resource_group.aks.location From c4ef3e1bbe6b7804bfc88c37a7690ed07926a323 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 28 Mar 2021 07:30:03 +0100 Subject: [PATCH 19/58] Changed readme title --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f26b023..c85ac340 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# aks-nexus-velero +# DevOps Lab [![Build environment](https://github.com/adamrushuk/devops-lab/workflows/build/badge.svg)](https://github.com/adamrushuk/devops-lab/actions?query=workflow%3A%22build) From 39f380695f1a3e2f261fc5f47dfee6795c3b3b67 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 28 Mar 2021 07:32:52 +0100 Subject: [PATCH 20/58] Bumped tflint versions --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b1bd8b3b..ebd983b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,9 +60,9 @@ env: TF_VERSION: "0.14.9" # "latest" is supported TF_WORKING_DIR: ./terraform # https://github.com/terraform-linters/tflint-ruleset-azurerm/releases - TFLINT_RULESET_AZURERM_VERSION: "v0.8.2" + TFLINT_RULESET_AZURERM_VERSION: "v0.9.0" # https://github.com/terraform-linters/tflint/releases - TFLINT_VERSION: "v0.24.1" + TFLINT_VERSION: "v0.25.0" # Env var concatenation is currently not supported at Workflow or Job scope. See workaround below: # https://github.community/t5/GitHub-Actions/How-can-we-concatenate-multiple-env-vars-at-workflow-and-job/td-p/48489 From bc6b918314492fa3e79e1f95f1e9145206405297 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sun, 28 Mar 2021 07:41:53 +0100 Subject: [PATCH 21/58] Removed azurerm_log_analytics_workspace_invalid_retention_in_days tflint rule --- scripts/tflint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tflint.sh b/scripts/tflint.sh index 77e5ea19..57482d98 100644 --- a/scripts/tflint.sh +++ b/scripts/tflint.sh @@ -14,7 +14,7 @@ TFLINT_RULESET_AZURERM_VERSION="${TFLINT_RULESET_AZURERM_VERSION:-v0.7.0}" TF_FLAGS=("$TF_WORKING_DIR") export TFLINT_LOG=debug # use empty array to skip adding disabled rules, eg: "DISABLED_RULES=()" -DISABLED_RULES=("azurerm_log_analytics_workspace_invalid_retention_in_days") +DISABLED_RULES=() # use dynamic flags if [ ${#DISABLED_RULES[@]} -gt 0 ]; then From a95402df1e6852222be22294ef9effeffb09da33 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 30 Mar 2021 07:49:43 +0100 Subject: [PATCH 22/58] Changed tf helm filenames --- terraform/{helm_aad_pod_identity.tf => aad_pod_identity_helm.tf} | 0 terraform/{helm_akv2k8s.tf => akv2k8s_helm.tf} | 0 terraform/{helm_argocd.tf => argocd_helm.tf} | 0 terraform/{helm_cert_manager.tf => cert_manager_helm.tf} | 0 terraform/{helm_gitlab.tf => gitlab_helm.tf} | 0 terraform/{helm_kured.tf => kured_helm.tf} | 0 terraform/{helm_nexus.tf => nexus_helm.tf} | 0 terraform/{helm_nginx.tf => nginx_helm.tf} | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename terraform/{helm_aad_pod_identity.tf => aad_pod_identity_helm.tf} (100%) rename terraform/{helm_akv2k8s.tf => akv2k8s_helm.tf} (100%) rename terraform/{helm_argocd.tf => argocd_helm.tf} (100%) rename terraform/{helm_cert_manager.tf => cert_manager_helm.tf} (100%) rename terraform/{helm_gitlab.tf => gitlab_helm.tf} (100%) rename terraform/{helm_kured.tf => kured_helm.tf} (100%) rename terraform/{helm_nexus.tf => nexus_helm.tf} (100%) rename terraform/{helm_nginx.tf => nginx_helm.tf} (100%) diff --git a/terraform/helm_aad_pod_identity.tf b/terraform/aad_pod_identity_helm.tf similarity index 100% rename from terraform/helm_aad_pod_identity.tf rename to terraform/aad_pod_identity_helm.tf diff --git a/terraform/helm_akv2k8s.tf b/terraform/akv2k8s_helm.tf similarity index 100% rename from terraform/helm_akv2k8s.tf rename to terraform/akv2k8s_helm.tf diff --git a/terraform/helm_argocd.tf b/terraform/argocd_helm.tf similarity index 100% rename from terraform/helm_argocd.tf rename to terraform/argocd_helm.tf diff --git a/terraform/helm_cert_manager.tf b/terraform/cert_manager_helm.tf similarity index 100% rename from terraform/helm_cert_manager.tf rename to terraform/cert_manager_helm.tf diff --git a/terraform/helm_gitlab.tf b/terraform/gitlab_helm.tf similarity index 100% rename from terraform/helm_gitlab.tf rename to terraform/gitlab_helm.tf diff --git a/terraform/helm_kured.tf b/terraform/kured_helm.tf similarity index 100% rename from terraform/helm_kured.tf rename to terraform/kured_helm.tf diff --git a/terraform/helm_nexus.tf b/terraform/nexus_helm.tf similarity index 100% rename from terraform/helm_nexus.tf rename to terraform/nexus_helm.tf diff --git a/terraform/helm_nginx.tf b/terraform/nginx_helm.tf similarity index 100% rename from terraform/helm_nginx.tf rename to terraform/nginx_helm.tf From 46ccc64447bca5893687e3cd305737436c707e5c Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 30 Mar 2021 07:50:35 +0100 Subject: [PATCH 23/58] Added argocd oidc sso config --- terraform/argocd_sso.tf | 100 ++++++++++++++++++++++++++++++++++++++++ terraform/variables.tf | 10 ++++ 2 files changed, 110 insertions(+) create mode 100644 terraform/argocd_sso.tf diff --git a/terraform/argocd_sso.tf b/terraform/argocd_sso.tf new file mode 100644 index 00000000..64137640 --- /dev/null +++ b/terraform/argocd_sso.tf @@ -0,0 +1,100 @@ +# Configures Azure AD App Registration Auth using OIDC +# +# https://argo-cd.readthedocs.io/en/stable/operator-manual/user-management/microsoft/#azure-ad-app-registration-auth-using-oidc + +resource "random_password" "argocd" { + length = 32 + special = false + keepers = { + service_principal = azuread_application.argocd.id + } +} + +# https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/application +resource "azuread_application" "argocd" { + display_name = var.argocd_app_reg_name + prevent_duplicate_names = true + homepage = "https://${var.argocd_fqdn}" + identifier_uris = ["https://${var.argocd_app_reg_name}"] + reply_urls = ["https://${var.argocd_fqdn}/auth/callback"] + available_to_other_tenants = false + oauth2_allow_implicit_flow = false + # owners = [] + group_membership_claims = "All" + + # you can check manually created app reg info in the app reg manifest tab + # reference: https://github.com/mjisaak/azure-active-directory/blob/master/README.md#well-known-appids + required_resource_access { + # Microsoft Graph + resource_app_id = "00000003-0000-0000-c000-000000000000" + + # User.Read - e1fe6dd8-ba31-4d61-89e7-88639da4683d - Sign in and read user profile + resource_access { + id = "e1fe6dd8-ba31-4d61-89e7-88639da4683d" + type = "Scope" + } + } + + optional_claims { + access_token { + name = "groups" + source = null + essential = false + additional_properties = [] + } + + id_token { + name = "groups" + source = null + essential = false + additional_properties = [] + } + } +} + +# https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/application_password +resource "azuread_application_password" "argocd" { + application_object_id = azuread_application.argocd.id + description = "argocd_secret" + value = random_password.argocd.result + end_date = "2099-01-01T01:02:03Z" +} + +# TODO: add "SelfServiceAppAccess" tag to enable self-service options in Enterprise App +resource "azuread_service_principal" "argocd" { + application_id = azuread_application.argocd.application_id +} + +data "azurerm_client_config" "current" { +} + +# https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file +data "template_file" "argocd_cm" { + template = file(var.argocd_cm_yaml_path) + vars = { + tenantId = data.azurerm_client_config.current.tenant_id + appClientId = azuread_service_principal.argocd.application_id + } +} + +# https://www.terraform.io/docs/provisioners/local-exec.html +resource "null_resource" "argocd_cm" { + triggers = { + yaml_contents = filemd5(var.argocd_cm_yaml_path) + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = var.aks_config_path + } + command = < Date: Tue, 30 Mar 2021 07:59:29 +0100 Subject: [PATCH 24/58] Added argocd-cm patch template --- terraform/files/argocd-cm-patch.tmpl.yaml | 13 +++++++++++ terraform/files/argocd-gitlab.yaml | 28 ----------------------- terraform/files/argocd-jenkins.yaml | 24 ------------------- terraform/variables.tf | 7 +----- 4 files changed, 14 insertions(+), 58 deletions(-) create mode 100644 terraform/files/argocd-cm-patch.tmpl.yaml delete mode 100644 terraform/files/argocd-gitlab.yaml delete mode 100644 terraform/files/argocd-jenkins.yaml diff --git a/terraform/files/argocd-cm-patch.tmpl.yaml b/terraform/files/argocd-cm-patch.tmpl.yaml new file mode 100644 index 00000000..208dcfa7 --- /dev/null +++ b/terraform/files/argocd-cm-patch.tmpl.yaml @@ -0,0 +1,13 @@ +data: + oidc.config: | + name: Azure + issuer: https://login.microsoftonline.com/${tenantId}/v2.0 + clientID: ${appClientId} + clientSecret: $oidc.azure.clientSecret + requestedIDTokenClaims: + groups: + essential: true + requestedScopes: + - openid + - profile + - email diff --git a/terraform/files/argocd-gitlab.yaml b/terraform/files/argocd-gitlab.yaml deleted file mode 100644 index dbdbe0aa..00000000 --- a/terraform/files/argocd-gitlab.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# https://argoproj.github.io/argo-cd/user-guide/helm/ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - # ! MUST match the existing helm release name: https://argoproj.github.io/argo-cd/user-guide/helm/#helm-release-name - name: ar-gitlab - # namespace of argo cd deployment - namespace: argocd -spec: - project: default - source: - repoURL: git@github.com:adamrushuk/charts-private.git - targetRevision: main - path: charts/gitlab-https - helm: - # target helm version - version: v3 - # values file path is relative from the source.path folder - valueFiles: - - ar-values-dev.yaml - destination: - server: https://kubernetes.default.svc - namespace: gitlab - syncPolicy: {} - # # sync options which modifies sync behavior - # syncOptions: - # # namespace Auto-Creation ensures that namespace specified as the application destination exists in the destination cluster - # - CreateNamespace=true diff --git a/terraform/files/argocd-jenkins.yaml b/terraform/files/argocd-jenkins.yaml deleted file mode 100644 index 3dfc0293..00000000 --- a/terraform/files/argocd-jenkins.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# https://argoproj.github.io/argo-cd/user-guide/helm/ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - # ! MUST match the existing helm release name - name: ar-jenkins - namespace: argocd -spec: - project: default - source: - repoURL: git@github.com:adamrushuk/charts-private.git - targetRevision: main - path: jenkins - helm: - # target helm version - version: v3 - # releaseName: ar-jenkins - valueFiles: - - ar-values-dev.yaml - destination: - server: https://kubernetes.default.svc - namespace: jenkins - syncPolicy: {} diff --git a/terraform/variables.tf b/terraform/variables.tf index 3cbd383e..9c0c9bd6 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -350,7 +350,7 @@ variable "argocd_app_reg_name" { } variable "argocd_cm_yaml_path" { - default = "argocd-cm-patch.tmpl.yaml" + default = "files/argocd-cm-patch.tmpl.yaml" } @@ -359,8 +359,3 @@ variable "argocd_cm_yaml_path" { variable "gitlab_cert_sync_yaml_path" { default = "files/gitlab-akvs-certificate-sync.yaml" } - -# TODO: remove if no longer required -# variable "gitlab_argocd_app_path" { -# default = "files/argocd-gitlab.yaml" -# } From 9eda05b118ce9559bb6d32fc7536d03e293f3d35 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 30 Mar 2021 08:44:05 +0100 Subject: [PATCH 25/58] Added argocd secret patch --- terraform/argocd_sso.tf | 38 ++++++++++++++++++- terraform/files/argocd-secret-patch.tmpl.yaml | 2 + terraform/variables.tf | 4 ++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 terraform/files/argocd-secret-patch.tmpl.yaml diff --git a/terraform/argocd_sso.tf b/terraform/argocd_sso.tf index 64137640..e9a190f2 100644 --- a/terraform/argocd_sso.tf +++ b/terraform/argocd_sso.tf @@ -20,7 +20,7 @@ resource "azuread_application" "argocd" { available_to_other_tenants = false oauth2_allow_implicit_flow = false # owners = [] - group_membership_claims = "All" + group_membership_claims = "All" # you can check manually created app reg info in the app reg manifest tab # reference: https://github.com/mjisaak/azure-active-directory/blob/master/README.md#well-known-appids @@ -68,6 +68,8 @@ resource "azuread_service_principal" "argocd" { data "azurerm_client_config" "current" { } + +# argocd-cm patch # https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file data "template_file" "argocd_cm" { template = file(var.argocd_cm_yaml_path) @@ -81,6 +83,7 @@ data "template_file" "argocd_cm" { resource "null_resource" "argocd_cm" { triggers = { yaml_contents = filemd5(var.argocd_cm_yaml_path) + sp_app_id = azuread_service_principal.argocd.application_id } provisioner "local-exec" { @@ -98,3 +101,36 @@ resource "null_resource" "argocd_cm" { null_resource.argocd_configure ] } + + +# argocd-secret patch +# https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file +data "template_file" "argocd_secret" { + template = file(var.argocd_secret_yaml_path) + vars = { + clientSecretBase64 = base64encode(random_password.argocd.result) + } +} + +# https://www.terraform.io/docs/provisioners/local-exec.html +resource "null_resource" "argocd_secret" { + triggers = { + yaml_contents = filemd5(var.argocd_secret_yaml_path) + clientSecret = random_password.argocd.result + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = var.aks_config_path + } + command = < Date: Tue, 30 Mar 2021 08:52:22 +0100 Subject: [PATCH 26/58] Added escape char to argocd-cm template --- terraform/files/argocd-cm-patch.tmpl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/files/argocd-cm-patch.tmpl.yaml b/terraform/files/argocd-cm-patch.tmpl.yaml index 208dcfa7..3473549c 100644 --- a/terraform/files/argocd-cm-patch.tmpl.yaml +++ b/terraform/files/argocd-cm-patch.tmpl.yaml @@ -3,7 +3,7 @@ data: name: Azure issuer: https://login.microsoftonline.com/${tenantId}/v2.0 clientID: ${appClientId} - clientSecret: $oidc.azure.clientSecret + clientSecret: \$oidc.azure.clientSecret requestedIDTokenClaims: groups: essential: true From b245d3341eee3f0a029dadb382c28540d21ae274 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Tue, 30 Mar 2021 09:07:16 +0100 Subject: [PATCH 27/58] Added argocd-rbac-cm patch --- terraform/argocd_sso.tf | 38 +++++++++++++++++++ .../files/argocd-rbac-cm-patch.tmpl.yaml | 5 +++ terraform/variables.tf | 8 ++++ 3 files changed, 51 insertions(+) create mode 100644 terraform/files/argocd-rbac-cm-patch.tmpl.yaml diff --git a/terraform/argocd_sso.tf b/terraform/argocd_sso.tf index e9a190f2..6fd9925a 100644 --- a/terraform/argocd_sso.tf +++ b/terraform/argocd_sso.tf @@ -134,3 +134,41 @@ resource "null_resource" "argocd_secret" { null_resource.argocd_configure ] } + + +# argocd-rbac-cm patch +data "azuread_group" "argocd_admins" { + display_name = var.argocd_admins_aad_group_name + security_enabled = true +} + +# https://registry.terraform.io/providers/hashicorp/template/latest/docs/data-sources/file +data "template_file" "argocd_rbac_cm" { + template = file(var.argocd_rbac_cm_yaml_path) + vars = { + argoAdminGroupId = azuread_group.argocd_admins.id + } +} + +# https://www.terraform.io/docs/provisioners/local-exec.html +resource "null_resource" "argocd_rbac_cm" { + triggers = { + yaml_contents = filemd5(var.argocd_rbac_cm_yaml_path) + argoAdminGroupId = azuread_group.argocd_admins.id + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = var.aks_config_path + } + command = < Date: Tue, 30 Mar 2021 09:12:44 +0100 Subject: [PATCH 28/58] Fixed argocd_admins data resource --- terraform/argocd_sso.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/argocd_sso.tf b/terraform/argocd_sso.tf index 6fd9925a..17aed967 100644 --- a/terraform/argocd_sso.tf +++ b/terraform/argocd_sso.tf @@ -146,7 +146,7 @@ data "azuread_group" "argocd_admins" { data "template_file" "argocd_rbac_cm" { template = file(var.argocd_rbac_cm_yaml_path) vars = { - argoAdminGroupId = azuread_group.argocd_admins.id + argoAdminGroupId = data.azuread_group.argocd_admins.id } } @@ -154,7 +154,7 @@ data "template_file" "argocd_rbac_cm" { resource "null_resource" "argocd_rbac_cm" { triggers = { yaml_contents = filemd5(var.argocd_rbac_cm_yaml_path) - argoAdminGroupId = azuread_group.argocd_admins.id + argoAdminGroupId = data.azuread_group.argocd_admins.id } provisioner "local-exec" { From 258c1abe155ab9fd65300b174f84a5e097f09c75 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 1 Apr 2021 07:16:58 +0100 Subject: [PATCH 29/58] Added log output mask --- terraform/argocd_sso.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/argocd_sso.tf b/terraform/argocd_sso.tf index 17aed967..333d3107 100644 --- a/terraform/argocd_sso.tf +++ b/terraform/argocd_sso.tf @@ -125,6 +125,9 @@ resource "null_resource" "argocd_secret" { KUBECONFIG = var.aks_config_path } command = < Date: Thu, 1 Apr 2021 07:31:15 +0100 Subject: [PATCH 30/58] Removed aks-preview extension --- scripts/start_aks_cluster.sh | 8 ++------ scripts/stop_aks_cluster.sh | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/scripts/start_aks_cluster.sh b/scripts/start_aks_cluster.sh index d7f7a6bf..0f0bae5a 100644 --- a/scripts/start_aks_cluster.sh +++ b/scripts/start_aks_cluster.sh @@ -6,12 +6,8 @@ set -euo pipefail trap "echo 'error: Script failed: see failed command above'" ERR -# Prereqs as this is a preview feature: https://docs.microsoft.com/en-us/azure/aks/start-stop-cluster -# Install the aks-preview extension -az extension add --name aks-preview - -# Update the extension to make sure you have the latest version installed -az extension update --name aks-preview +# info +az version # Check AKS power state aks_power_state=$(az aks show --name "$AKS_CLUSTER_NAME" --resource-group "$AKS_RG_NAME" --output tsv --query 'powerState.code') diff --git a/scripts/stop_aks_cluster.sh b/scripts/stop_aks_cluster.sh index 83bd3e55..7a64b149 100644 --- a/scripts/stop_aks_cluster.sh +++ b/scripts/stop_aks_cluster.sh @@ -6,12 +6,8 @@ set -euo pipefail trap "echo 'error: Script failed: see failed command above'" ERR -# Prereqs as this is a preview feature: https://docs.microsoft.com/en-us/azure/aks/start-stop-cluster -# Install the aks-preview extension -az extension add --name aks-preview - -# Update the extension to make sure you have the latest version installed -az extension update --name aks-preview +# info +az version # Check AKS power state aks_power_state=$(az aks show --name "$AKS_CLUSTER_NAME" --resource-group "$AKS_RG_NAME" --output tsv --query 'powerState.code') From 4965790f3d1ff44b6a17acb82af65d144522d1a1 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 1 Apr 2021 07:36:07 +0100 Subject: [PATCH 31/58] Removed old SAML SSO config / notes --- terraform/argocd_sso/README.md | 211 ------------------ .../argocd-cm-sso-patch.TEMPLATE.yaml | 25 --- .../argocd-rbac-cm-patch.TEMPLATE.yaml | 17 -- terraform/argocd_sso/argocd_aad.tf | 145 ------------ terraform/argocd_sso/sso_claims.sh | 62 ----- terraform/argocd_sso/tf_test.ps1 | 16 -- 6 files changed, 476 deletions(-) delete mode 100644 terraform/argocd_sso/README.md delete mode 100644 terraform/argocd_sso/argocd-cm-sso-patch.TEMPLATE.yaml delete mode 100644 terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml delete mode 100644 terraform/argocd_sso/argocd_aad.tf delete mode 100644 terraform/argocd_sso/sso_claims.sh delete mode 100644 terraform/argocd_sso/tf_test.ps1 diff --git a/terraform/argocd_sso/README.md b/terraform/argocd_sso/README.md deleted file mode 100644 index 8152b384..00000000 --- a/terraform/argocd_sso/README.md +++ /dev/null @@ -1,211 +0,0 @@ - -# Argo CD Notes - -A collection of notes whilst testing Argo CD. - -Full SSO configuration currently cannot be done with Terraform, so I've partial automated the Application Registration, -and it's Service Principle (which makes an "Enterprise App"), but there are manual steps afterwards: - -- Add `Sign on URL` -- Add `email` User Claim -- Create `SAML Signing Cert` -- Download SAML cert (base64), ready for the ConfigMap yaml -- Create yaml ConfigMaps for SSO and RBAC -- Apply ConfigMaps - - -## Contents - -- [Reference](#reference) -- [Getting Started](#getting-started) -- [Add Repository](#add-repository) -- [Configure SSO for Argo CD](#configure-sso-for-argo-cd) - -## Reference - -- https://github.com/argoproj/argo-cd/blob/master/docs/faq.md#i-forgot-the-admin-password-how-do-i-reset-it - -## Getting Started - -Use `--grpc-web` if you get the `argocd transport: received the unexpected content-type "text/plain; charset=utf-8"` error - -```bash -# vars -ARGO_SERVER="argocd.thehypepipe.co.uk" - -# install -VERSION=$(curl --silent "https://api.github.com/repos/argoproj/argo-cd/releases/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/') -sudo curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/download/$VERSION/argocd-linux-amd64 -sudo chmod +x /usr/local/bin/argocd - -# show version -argocd version --grpc-web --server "$ARGO_SERVER" - -# get admin password -# default password is server pod name, eg: "argocd-server-89c6cd7d4-h7vmn" -ARGO_ADMIN_PASSWORD=$(kubectl get pods -n argocd -l app.kubernetes.io/name=argocd-server -o name | cut -d'/' -f 2) - -# login -argocd logout -h -argocd logout "$ARGO_SERVER" -argocd login -h -argocd login "$ARGO_SERVER" --grpc-web --username admin --password "$ARGO_ADMIN_PASSWORD" - -# change password -read -s NEW_ARGO_ADMIN_PASSWORD -# echo "$NEW_ARGO_ADMIN_PASSWORD" -argocd account update-password --grpc-web -h -argocd account update-password --grpc-web --account admin --current-password "$ARGO_ADMIN_PASSWORD" --new-password "$NEW_ARGO_ADMIN_PASSWORD" - -# test new admin password -argocd logout "$ARGO_SERVER" -argocd login "$ARGO_SERVER" --grpc-web --username admin --password "$NEW_ARGO_ADMIN_PASSWORD" - -# account tasks -argocd account list -argocd account -h - -# misc -argocd -h -``` - -## Add Repository - -```bash -# Add a Git repository via SSH using a private key for authentication, ignoring the server's host key -# argocd repo add git@github.com:adamrushuk/charts-private.git --insecure-ignore-host-key --ssh-private-key-path ~/.ssh/id_ed25519 -argocd repo add -h -argocd repo add git@github.com:adamrushuk/charts-private.git --ssh-private-key-path ~/.ssh/id_ed25519 - -# add known_host entries for private git server -ssh-keyscan gitlab.thehypepipe.co.uk | argocd cert add-ssh --batch - -# create ssh key for private git repo access -# ~/.ssh/id_ed25519 -ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_argocd -C "argocd@gitlab.thehypepipe.co.uk" -ll ~/.ssh - -# check public key fingerprint -ssh-keygen -lf ~/.ssh/id_ed25519_argocd.pub - -# copy public key and enter into source git repo settings -# eg, GitLab repo deploy key: https://gitlab.thehypepipe.co.uk/helm-charts/-/settings/repository > Deploy Keys -cat ~/.ssh/id_ed25519_argocd.pub - -# add helm chart repository -argocd repo add git@gitlab.thehypepipe.co.uk/helm-charts.git --ssh-private-key-path ~/.ssh/id_ed25519_argocd - -# show repo -argocd repo list -``` - -## Configure SSO for Argo CD - -https://argoproj.github.io/argo-cd/operator-manual/user-management/microsoft/ - -```bash -# subscription where ArgoCD is deployed -AR-Dev - -# created new AAD groups, eg: -AR-Dev_ArgoCD_Admin -AR-Dev_ArgoCD_ReadOnly - -# created argo enterprise app -AR-Dev_ArgoCD - - -# Basic SAML Configuration -# Identifier (Entity ID) -https://argocd.thehypepipe.co.uk/api/dex/callback -# Reply URL (Assertion Consumer Service URL) -https://argocd.thehypepipe.co.uk/api/dex/callback -# Sign on URL -https://argocd.thehypepipe.co.uk/auth/login - -# User Attributes & Claims -# + Add new claim | Name: email | Source: Attribute | Source attribute: user.userprincipalname -+ Add new claim | Name: email | Source: Attribute | Source attribute: user.primaryauthoritativeemail - -+ Add group claim | Which groups: All groups | Source attribute: Group ID | Customize: True | Name: Group | Namespace: | Emit groups as role claims: False - -# Create a "Sign SAML assertion" SAML Signing Cert (SHA-256) -# Download and base64 the cert, ready for the ConfigMap yaml - -# Login URL (ssoURL) -https://login.microsoftonline.com//saml2 -# Azure AD Identifier -https://sts.windows.net// -# Logout URL -https://login.microsoftonline.com//saml2 - - -# SSO: User Attributes & Claims -# select user.userprincipalname instead of user.mail -+ Add new claim | Name: email | Source: Attribute | Source attribute: user.userprincipalname - - - - -## Create RBAC patch ## -# RBAC vars -ARGO_ADMIN_GROUP_NAME="AR-Dev_ArgoCD_Admins" -ARGO_ADMIN_GROUP_ID=$(az ad group show --group "$ARGO_ADMIN_GROUP_NAME" --query "objectId" --output tsv) - -# Create RBAC patch yaml -cat > argocd-rbac-cm-patch.yaml << EOF -# Patch ConfigMap to add RBAC config -data: - policy.default: role:readonly - - # Map AAD Group Object Id to an Argo CD role - # (Nested groups work fine) - # g, , role:admin - policy.csv: | - g, $ARGO_ADMIN_GROUP_ID, role:admin -EOF - -# Apply yaml RBAC patch for default admin and readonly roles -kubectl patch configmap/argocd-rbac-cm --namespace argocd --type merge --patch "$(cat argocd-rbac-cm-patch.yaml)" - - - -## Create SSO patch yaml ## -# SSO vars -ARGO_FQDN="argocd.thehypepipe.co.uk" -TENANT_ID=$(az account show --query "tenantId" --output tsv) -# assumes SAML Signing Certificate has been downloaded/saved as "ArgoCD.cer" (choosing Certificate (Base64) option) -SAML_CERT_BASE64=$(cat ArgoCD.cer | base64) -echo "$SAML_CERT_BASE64" - -# created indented string ready for caData YAML multi-line block -SAML_CERT_BASE64_INDENTED=$(cat ArgoCD.cer | base64 | sed 's/^/ /') -echo "$SAML_CERT_BASE64_INDENTED" - -cat > argocd-cm-sso-patch.yaml << EOF -# Patch ConfigMap to add dex SSO config -# source: https://argoproj.github.io/argo-cd/operator-manual/user-management/microsoft/ -data: - dex.config: | - logger: - level: debug - format: json - connectors: - - type: saml - id: saml - name: saml - config: - entityIssuer: https://$ARGO_FQDN/api/dex/callback - ssoURL: https://login.microsoftonline.com/$TENANT_ID/saml2 - caData: | -$SAML_CERT_BASE64_INDENTED - redirectURI: https://$ARGO_FQDN/api/dex/callback - usernameAttr: email - emailAttr: email - groupsAttr: Group -EOF - -# Apply SSO patch -kubectl patch configmap/argocd-cm --namespace argocd --type merge --patch "$(cat argocd-cm-sso-patch.yaml)" - -``` diff --git a/terraform/argocd_sso/argocd-cm-sso-patch.TEMPLATE.yaml b/terraform/argocd_sso/argocd-cm-sso-patch.TEMPLATE.yaml deleted file mode 100644 index b32c7316..00000000 --- a/terraform/argocd_sso/argocd-cm-sso-patch.TEMPLATE.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Patch ConfigMap to add dex SSO config -# source: https://argoproj.github.io/argo-cd/operator-manual/user-management/microsoft/ -# -# After following steps above, modify content and run command below to apply patch: -# kubectl patch configmap/argocd-cm --namespace argocd --type merge --patch "$(cat argocd-cm-patch-dev.yaml)" - -# TEMPLATE -data: - dex.config: | - logger: - level: debug - format: json - connectors: - - type: saml - id: saml - name: saml - config: - entityIssuer: https://$ARGO_FQDN/api/dex/callback - ssoURL: https://login.microsoftonline.com/$TENANT_ID/saml2 - caData: | - $SAML_CERT_BASE64 - redirectURI: https://$ARGO_FQDN/api/dex/callback - usernameAttr: email - emailAttr: email - groupsAttr: Group diff --git a/terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml b/terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml deleted file mode 100644 index 33931260..00000000 --- a/terraform/argocd_sso/argocd-rbac-cm-patch.TEMPLATE.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Patch ConfigMap to add RBAC config -# source: -# - https://argoproj.github.io/argo-cd/operator-manual/rbac/ -# -# Run command below to apply patch for default admin and readonly roles: -# kubectl patch configmap/argocd-rbac-cm --namespace argocd --type merge --patch-file "argocd-rbac-cm-patch.yaml" - -# TEMPLATE -data: - policy.default: role:readonly - - # Map AAD Group Object Id to an Argo CD role - # (Nested groups work fine) - # g, , role:admin - policy.csv: | - g, admin, role:admin - g, $ARGO_ADMIN_GROUP_ID, role:admin diff --git a/terraform/argocd_sso/argocd_aad.tf b/terraform/argocd_sso/argocd_aad.tf deleted file mode 100644 index e4131573..00000000 --- a/terraform/argocd_sso/argocd_aad.tf +++ /dev/null @@ -1,145 +0,0 @@ -provider "azurerm" { - version = "2.44.0" - features {} -} - -variable "dns_zone_name" { - default = "thehypepipe.co.uk" -} - -variable "admin_consent" { - default = true -} - - -# TODO: remove temp outputs -# data "azuread_application" "argocd_manual" { -# display_name = "AR-Dev_ArgoCD" -# } - -# output "azure_app_object_manual" { -# value = data.azuread_application.argocd_manual -# } - -# data "azuread_service_principal" "argocd_manual" { -# display_name = "AR-Dev_ArgoCD" -# } - -# output "azure_sp_object_manual" { -# value = data.azuread_service_principal.argocd_manual -# } - -output "azure_ad_object_argocd" { - value = azuread_application.argocd -} -output "azure_sp_object_argocd" { - value = azuread_service_principal.argocd -} - -# https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/application -# source: https://github.com/hashicorp/terraform-provider-azuread/issues/173#issuecomment-663727531 -resource "azuread_application" "argocd" { - display_name = "ArgoCD" - prevent_duplicate_names = true - homepage = "https://argocd.${var.dns_zone_name}" - identifier_uris = ["https://argocd.${var.dns_zone_name}/api/dex/callback"] - reply_urls = ["https://argocd.${var.dns_zone_name}/api/dex/callback"] - available_to_other_tenants = false - oauth2_allow_implicit_flow = true - # type = "webapp/api" - # owners = ["00000004-0000-0000-c000-000000000000"] - group_membership_claims = "All" - - required_resource_access { - //https://docs.microsoft.com/en-us/azure/active-directory/manage-apps/grant-admin-consent - resource_app_id = "00000003-0000-0000-c000-000000000000" - resource_access { - id = "5f8c59db-677d-491f-a6b8-5f174b11ec1d" - type = "Scope" - } - resource_access { - id = "e1fe6dd8-ba31-4d61-89e7-88639da4683d" - type = "Scope" - } - } - - app_role { - allowed_member_types = [ - "User" - ] - - description = "User" - display_name = "User" - is_enabled = true - } - - app_role { - allowed_member_types = [ - "User" - ] - - description = "msiam_access" - display_name = "msiam_access" - is_enabled = true - } - - // We need to wait because Azure Graph API returns a 200 before its call-able #eventualconsistancy... - provisioner "local-exec" { - command = "sleep 20" - } - - //https://github.com/Azure/azure-cli/issues/7579 - //Add metadata URL - // provisioner "local-exec" { - // command = "az ad app update --id ${self.application_id} --set samlMetadataUrl=${var.saml_metadata_url}" - // } - // We need to wait because Azure Graph API returns a 200 before its call-able #eventualconsistancy... - // provisioner "local-exec" { - // command = "sleep 5" - // } - //https://github.com/Azure/azure-cli/issues/12946 - //https://github.com/Azure/azure-cli/issues/11534 - //https://docs.microsoft.com/en-us/azure/active-directory/develop/active-directory-optional-claims - //Optional Claims for tokens - provisioner "local-exec" { - command = "az rest --method PATCH --uri 'https://graph.microsoft.com/v1.0/applications/${self.object_id}' --body '{\"optionalClaims\": {\"saml2Token\": [{\"name\": \"groups\", \"additionalProperties\": []}]}}'" - } -} - -resource "azuread_service_principal" "argocd" { - //https://github.com/Azure/azure-cli/issues/9250 - application_id = azuread_application.argocd.application_id - tags = [ - "WindowsAzureActiveDirectoryIntegratedApp", - "WindowsAzureActiveDirectoryCustomSingleSignOnApplication", - "WindowsAzureActiveDirectoryGalleryApplicationNonPrimaryV1" - ] - - // We need to wait because Azure Graph API returns a 200 before its call-able #eventualconsistancy... - provisioner "local-exec" { - command = "sleep 20" - } - - # https://docs.microsoft.com/en-us/graph/application-saml-sso-configure-api?tabs=http#set-single-sign-on-mode - provisioner "local-exec" { - command = "az ad sp update --id ${azuread_application.argocd.application_id} --set preferredSingleSignOnMode='saml'" - } - - # depends_on = [ - # azuread_application.argocd - # ] -} - -resource "null_resource" "grant_admin_constent" { - count = var.admin_consent ? 1 : 0 - // https://docs.microsoft.com/en-us/cli/azure/ad/app/permission?view=azure-cli-latest#code-try-3 - provisioner "local-exec" { - command = "sleep 20" - } - provisioner "local-exec" { - command = "az ad app permission admin-consent --id ${azuread_application.argocd.application_id}" - } - depends_on = [ - azuread_service_principal.argocd - ] -} diff --git a/terraform/argocd_sso/sso_claims.sh b/terraform/argocd_sso/sso_claims.sh deleted file mode 100644 index 8b09dbe8..00000000 --- a/terraform/argocd_sso/sso_claims.sh +++ /dev/null @@ -1,62 +0,0 @@ -# Adding extra SSO claim -# Set "ArgoCD" app reg () -ARGO_APP_OBJECT_ID="" -az rest --method PATCH --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" --body '{\"optionalClaims\": {\"saml2Token\": [{\"name\": \"test\", \"additionalProperties\": [\"sam_account_name\"]}]}}' - -az rest --method PATCH --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" --body '{\"optionalClaims\": {\"saml2Token\": [{\"name\": \"userprincipalname\", \"source\": \"user\", \"additionalProperties\": [\"email\"]}]}}' - -# works via PS -$ARGO_APP_OBJECT_ID = "" -az rest --method PATCH --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" --body '{\"optionalClaims\": {\"saml2Token\": [{\"name\": \"groups\", \"additionalProperties\": [\"sam_account_name\"]}]}}' - -# add custom email claim -az rest --method PATCH --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" --body '{\"optionalClaims\": {\"saml2Token\": [{\"name\": \"userprincipalname\", \"source\": \"user\", \"additionalProperties\": [\"email\"]}]}}' - -# add custom group claim -az rest --method PATCH --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" --body '{\"optionalClaims\": {\"saml2Token\": [{\"name\": \"groups\", \"source\": null}]}}' - - - -# Get -az rest --method GET --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" -az rest --method GET --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" | clip.exe - - "optionalClaims": { - "accessToken": [], - "idToken": [], - "saml2Token": [ - { - "additionalProperties": [], - "essential": false, - "name": "groups", - "source": null - } - ] - }, - - -# TF created "ArgoCD" App Reg -az rest --method GET --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" - - "optionalClaims": { - "accessToken": [], - "idToken": [], - "saml2Token": [ - { - "additionalProperties": [ - "sam_account_name" - ], - "essential": false, - "name": "test", - "source": null - } - ] - }, - -# AR-Dev_ArgoCD - App reg -az rest --method GET --uri "https://graph.microsoft.com/v1.0/applications/$ARGO_APP_OBJECT_ID" - -# manual "AR-Dev_ArgoCD" Enterprise App -SERVICE_PRINCIPLE_ID="" -az rest --method GET --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$SERVICE_PRINCIPLE_ID" -az rest --method GET --uri "https://graph.microsoft.com/v1.0/servicePrincipals/$SERVICE_PRINCIPLE_ID" | clip.exe diff --git a/terraform/argocd_sso/tf_test.ps1 b/terraform/argocd_sso/tf_test.ps1 deleted file mode 100644 index 7251a13c..00000000 --- a/terraform/argocd_sso/tf_test.ps1 +++ /dev/null @@ -1,16 +0,0 @@ -# testing Terraform config for Enterprise App -# use WSL -cd ./terraform/argocd_sso - -# login -az login -az account show - -# init -terraform init - -# apply -terraform apply - -# destroy -terraform destroy From 27fd681a90aa9083df5632789ffefed4365fb238 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Thu, 1 Apr 2021 09:33:47 +0100 Subject: [PATCH 32/58] Added notes on sealed-secrets usage --- sealed-secrets/README.md | 79 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 sealed-secrets/README.md diff --git a/sealed-secrets/README.md b/sealed-secrets/README.md new file mode 100644 index 00000000..10ddf439 --- /dev/null +++ b/sealed-secrets/README.md @@ -0,0 +1,79 @@ +# Sealed Secrets Notes + +[sealed-secrets](https://github.com/bitnami-labs/sealed-secrets) is a Kubernetes controller and tool for one-way +encrypted Secrets. + +**Problem**: "I can manage all my K8s config in git, except Secrets." + +**Solution**: Encrypt your Secret into a SealedSecret, which is safe to store - even to a public repository. The +SealedSecret can be decrypted only by the controller running in the target cluster and nobody else +(not even the original author) is able to obtain the original Secret from the SealedSecret. + +## Installation + +Before installation, consider reading the [Release Notes](https://github.com/bitnami-labs/sealed-secrets/blob/main/RELEASE-NOTES.md). + +### Helm Chart + +Use the code below to install the official [sealed-secrets helm chart](https://github.com/bitnami-labs/sealed-secrets/tree/main/helm/sealed-secrets): + +```bash +# add repo +helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets + +# list charts +helm search repo sealed-secrets + +# list all chart versions +helm search repo sealed-secrets/sealed-secrets -l + +# create namespace +kubectl create namespace sealed-secrets + +# install chart (dry-run) +helm upgrade sealed-secrets sealed-secrets/sealed-secrets --install --atomic --namespace sealed-secrets --debug --dry-run + +# install chart +helm upgrade sealed-secrets sealed-secrets/sealed-secrets --install --atomic --namespace sealed-secrets --debug + +# show status / notes +helm status sealed-secrets --namespace sealed-secrets +``` + +### Kubeseal CLI + +Install the kubeseal CLI by downloading the binary from [sealed-secrets/releases](https://github.com/bitnami-labs/sealed-secrets/releases). + +```bash +wget https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.15.0/kubeseal-linux-amd64 -O kubeseal +sudo install -m 755 kubeseal /usr/local/bin/kubeseal +``` + +## Usage + +The example below creates a secret, then uses kubeseal to encrypt it into a sealed-secret file. + +Once the sealed-secret file is applied into the kubernetes cluster, it is decrypted server-side to create a +standard secret in the target namespace. + +```bash +# create secret +# (note use of `--dry-run` - this is just a local file!) +echo -n SuperSecretPassw0rd | kubectl create secret generic mysecret --dry-run=client --from-literal=username=admin --from-file=password=/dev/stdin -o yaml > secret.yaml + +# create sealed-secret using stdin/stdout +kubeseal \ + --controller-namespace sealed-secrets \ + --controller-name sealed-secrets \ + --namespace my-target-namespace \ + < secret.yaml > sealed-secret.yaml + +# create namespace +kubectl create namespace my-target-namespace + +# apply sealed-secret +kubectl create --namespace my-target-namespace -f sealed-secret.yaml + +# show secret +kubectl get secret mysecret --namespace my-target-namespace -o yaml +``` From f91e6276977a9f1c39092c11db49ed08a2374b91 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 07:46:15 +0100 Subject: [PATCH 33/58] Bumped chart / provider versions --- terraform/variables.tf | 8 ++++---- terraform/versions.tf | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 953f5ad3..60dd2e48 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -21,7 +21,7 @@ variable "kubernetes_version" { # helm search repo ingress-nginx/ingress-nginx # * also update terraform/helm/nginx_values.yaml variable "nginx_chart_version" { - default = "3.24.0" + default = "3.26.0" } # https://hub.helm.sh/charts/jetstack/cert-manager @@ -54,7 +54,7 @@ variable "nexus_chart_version" { # https://github.com/SparebankenVest/public-helm-charts/blob/master/stable/akv2k8s/Chart.yaml#L5 # helm search repo spv-charts/akv2k8s variable "akv2k8s_chart_version" { - default = "2.0.7" + default = "2.0.10" } # https://github.com/Azure/aad-pod-identity/blob/master/charts/aad-pod-identity/Chart.yaml#L4 @@ -67,7 +67,7 @@ variable "aad_pod_identity_chart_version" { # https://github.com/bitnami/charts/blob/master/bitnami/external-dns/Chart.yaml#L21 # helm search repo bitnami/external-dns variable "external_dns_chart_version" { - default = "4.9.1" + default = "4.9.4" } # https://github.com/weaveworks/kured/tree/master/charts/kured @@ -86,7 +86,7 @@ variable "kured_image_tag" { # https://github.com/argoproj/argo-helm/blob/master/charts/argo-cd/Chart.yaml#L5 # helm search repo argo/argo-cd variable "argocd_chart_version" { - default = "2.17.1" + default = "2.17.5" } # https://hub.docker.com/r/argoproj/argocd/tags diff --git a/terraform/versions.tf b/terraform/versions.tf index 52cf6a4d..5f2745b2 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -19,7 +19,7 @@ terraform { # https://github.com/terraform-providers/terraform-provider-azurerm/releases azurerm = { source = "hashicorp/azurerm" - version = "~> 2.53.0" + version = "~> 2.54.0" } # https://github.com/terraform-providers/terraform-provider-azuread/releases @@ -37,7 +37,7 @@ terraform { # https://github.com/hashicorp/terraform-provider-helm/releases helm = { source = "hashicorp/helm" - version = "~> 2.0.3" + version = "~> 2.1.0" } random = { From c57a581d639e3455126f67ab8d784ebc60d96b7f Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 08:33:09 +0100 Subject: [PATCH 34/58] Testing helm diff --- terraform/helm/nexus_values.yaml | 4 ++-- terraform/kured_helm.tf | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/helm/nexus_values.yaml b/terraform/helm/nexus_values.yaml index 24d0c936..a1db129a 100644 --- a/terraform/helm/nexus_values.yaml +++ b/terraform/helm/nexus_values.yaml @@ -7,7 +7,7 @@ nexus: dockerPort: 5000 nexusPort: 8081 - storageSize: 8Gi + storageSize: 10Gi ## If defined, storageClassName: ## If set to "-", storageClassName: "", which disables dynamic provisioning @@ -24,4 +24,4 @@ image: repository: sonatype/nexus3 pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "3.27.0" + tag: "3.29.2" diff --git a/terraform/kured_helm.tf b/terraform/kured_helm.tf index f8ebfbba..0dace8a5 100644 --- a/terraform/kured_helm.tf +++ b/terraform/kured_helm.tf @@ -32,8 +32,8 @@ resource "helm_release" "kured" { } # increase testing period frequency, when testing with "sudo touch /var/run/reboot-required" - # set { - # name = "configuration.period" - # value = "1m" - # } + set { + name = "configuration.period" + value = "1m" + } } From a56a23fbc7c1b8b023b318055c75fc4d38cf68fb Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 08:40:34 +0100 Subject: [PATCH 35/58] Changed velero chart values for v2.15.0 --- terraform/helm/velero_values.yaml | 38 +++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/terraform/helm/velero_values.yaml b/terraform/helm/velero_values.yaml index 571b502f..a8065857 100644 --- a/terraform/helm/velero_values.yaml +++ b/terraform/helm/velero_values.yaml @@ -1,4 +1,4 @@ -# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.14.8/charts/velero/values.yaml +# source: https://github.com/vmware-tanzu/helm-charts/blob/velero-2.15.0/charts/velero/values.yaml ## ## Configuration settings that directly affect the Velero deployment YAML. @@ -18,6 +18,15 @@ image: imagePullSecrets: [] # - registrySecretName +# Annotations to add to the Velero deployment's. Optional. +# +# If you are using reloader use the following annotation with your VELERO_SECRET_NAME +annotations: {} +# secret.reloader.stakater.com/reload: "" + +# Labels to add to the Velero deployment's. Optional. +labels: {} + # Annotations to add to the Velero deployment's pod template. Optional. # # If using kube2iam or kiam, use the following annotation with your AWS_ACCOUNT_ID @@ -77,6 +86,11 @@ metrics: scrapeInterval: 30s scrapeTimeout: 10s + # service metdata if metrics are enabled + service: + annotations: {} + labels: {} + # Pod annotations for Prometheus podAnnotations: prometheus.io/scrape: "true" @@ -89,14 +103,6 @@ metrics: # ServiceMonitor namespace. Default to Velero namespace. # namespace: -# Install CRDs as a templates. Enabled by default. -installCRDs: true - -# Enable/disable all helm hooks annotations -# You should disable this if using a deploy tool that doesn't support helm hooks, -# such as ArgoCD -enableHelmHooks: true - ## ## End of deployment-related settings. ## @@ -173,6 +179,11 @@ configuration: restoreResourcePriorities: # `velero server` default: false restoreOnlyMode: + # `velero server` default: 20.0 + clientQPS: + # `velero server` default: 30 + clientBurst: + # # additional key/value pairs to be used as environment variables such as "AWS_CLUSTER_NAME: 'yourcluster.domain.tld'" extraEnvVars: {} @@ -262,6 +273,12 @@ restic: # Tolerations to use for the Restic daemonset. Optional. tolerations: [] + # Annotations to set for the Restic daemonset. Optional. + annotations: {} + + # labels to set for the Restic daemonset. Optional. + labels: {} + # Extra volumes for the Restic daemonset. Optional. extraVolumes: [] @@ -278,6 +295,9 @@ restic: securityContext: {} # fsGroup: 1337 + # Node selector to use for the Restic daemonset. Optional. + nodeSelector: {} + # Backup schedules to create. # Eg: # schedules: From 29a71a7826e76aadd338a828517ceb0789df1c38 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 08:40:51 +0100 Subject: [PATCH 36/58] Changed nexus default tag to v3.30.0 --- terraform/helm/nexus_values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/helm/nexus_values.yaml b/terraform/helm/nexus_values.yaml index a1db129a..ef675d17 100644 --- a/terraform/helm/nexus_values.yaml +++ b/terraform/helm/nexus_values.yaml @@ -7,7 +7,7 @@ nexus: dockerPort: 5000 nexusPort: 8081 - storageSize: 10Gi + storageSize: 8Gi ## If defined, storageClassName: ## If set to "-", storageClassName: "", which disables dynamic provisioning @@ -24,4 +24,4 @@ image: repository: sonatype/nexus3 pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "3.29.2" + tag: "3.30.0" From 65808a76a394a8f1a71a03b69b05c84134437be0 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 08:41:00 +0100 Subject: [PATCH 37/58] Removed configuration.period --- terraform/kured_helm.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/kured_helm.tf b/terraform/kured_helm.tf index 0dace8a5..f8ebfbba 100644 --- a/terraform/kured_helm.tf +++ b/terraform/kured_helm.tf @@ -32,8 +32,8 @@ resource "helm_release" "kured" { } # increase testing period frequency, when testing with "sudo touch /var/run/reboot-required" - set { - name = "configuration.period" - value = "1m" - } + # set { + # name = "configuration.period" + # value = "1m" + # } } From 1f16e78f24ae9ba68635ccc62fb66d161c68b316 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 08:54:11 +0100 Subject: [PATCH 38/58] Added experiments manifest diff feature for velero helm release --- terraform/velero.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/velero.tf b/terraform/velero.tf index 2b1318d8..a6354b3f 100644 --- a/terraform/velero.tf +++ b/terraform/velero.tf @@ -88,6 +88,10 @@ resource "helm_release" "velero" { timeout = 600 atomic = true + experiments { + manifest = true + } + values = [file("helm/velero_values.yaml")] set { From 0252159859019128dd2121e2f722553e5f9a5506 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 09:03:18 +0100 Subject: [PATCH 39/58] Moved experiments block to helm provider --- terraform/velero.tf | 4 ---- terraform/versions.tf | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/velero.tf b/terraform/velero.tf index a6354b3f..2b1318d8 100644 --- a/terraform/velero.tf +++ b/terraform/velero.tf @@ -88,10 +88,6 @@ resource "helm_release" "velero" { timeout = 600 atomic = true - experiments { - manifest = true - } - values = [file("helm/velero_values.yaml")] set { diff --git a/terraform/versions.tf b/terraform/versions.tf index 5f2745b2..74818cdc 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -89,4 +89,8 @@ provider "helm" { client_key = base64decode(module.aks.full_object.kube_admin_config[0].client_key) cluster_ca_certificate = base64decode(module.aks.full_object.kube_admin_config[0].cluster_ca_certificate) } + + experiments { + manifest = true + } } From c0146e232299d798e0e2a690afb89d0a98ef82ba Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 09:10:52 +0100 Subject: [PATCH 40/58] Removed experiments block in helm provider --- terraform/versions.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/terraform/versions.tf b/terraform/versions.tf index 74818cdc..1ae92542 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -90,7 +90,8 @@ provider "helm" { cluster_ca_certificate = base64decode(module.aks.full_object.kube_admin_config[0].cluster_ca_certificate) } - experiments { - manifest = true - } + # TODO: currently has issues when enabled + # experiments { + # manifest = true + # } } From da7aeaa01a6b33dc6d4cede623294e48b22f45f9 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 09:45:06 +0100 Subject: [PATCH 41/58] Changed add-mask for argocd null_resource --- terraform/argocd_sso.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/argocd_sso.tf b/terraform/argocd_sso.tf index 333d3107..ddbdff9f 100644 --- a/terraform/argocd_sso.tf +++ b/terraform/argocd_sso.tf @@ -126,7 +126,7 @@ resource "null_resource" "argocd_secret" { } command = < Date: Fri, 2 Apr 2021 09:59:31 +0100 Subject: [PATCH 42/58] Changed workflows (tidy up) --- .github/workflows/build.yml | 19 +++---------------- .github/workflows/destroy.yml | 3 ++- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ebd983b4..1e1deca3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -57,7 +57,8 @@ env: TF_IN_AUTOMATION: "true" TF_INPUT: "false" TF_PLAN: "tfplan" - TF_VERSION: "0.14.9" # "latest" is supported + # https://github.com/hashicorp/terraform/releases + TF_VERSION: "0.14.9" TF_WORKING_DIR: ./terraform # https://github.com/terraform-linters/tflint-ruleset-azurerm/releases TFLINT_RULESET_AZURERM_VERSION: "v0.9.0" @@ -155,10 +156,6 @@ jobs: run: terraform plan -out=${{ env.TF_PLAN }} working-directory: ${{ env.TF_WORKING_DIR }} - # - run: echo ${{ steps.plan.outputs.stdout }} - # - run: echo ${{ steps.plan.outputs.stderr }} - # - run: echo ${{ steps.plan.outputs.exitcode }} - - name: Terraform Apply run: ./scripts/terraform_apply.sh # only apply if changes are present @@ -177,13 +174,6 @@ jobs: # Ansible - # TODO: enable Ansible Lint once this issue has been resolved: https://github.com/ansible/ansible-lint-action/issues/36 - # - name: Lint Ansible Playbook - # uses: ansible/ansible-lint-action@6c8c141 - # with: - # targets: "./ansible" - - - name: Run Ansible playbook run: ./scripts/ansible.sh env: @@ -219,7 +209,6 @@ jobs: with: name: test results path: test/pester-test-results.xml - # TODO: only run when previous task (Pester tests) has been successful if: always() # remove NuGet proxy repo so pester report step doesnt fail @@ -228,9 +217,7 @@ jobs: # Shows at the bottom of a run: https://github.com/adamrushuk/devops-lab/runs/1035347513?check_suite_focus=true - name: Pester report - # TODO: remove continue-on-error once bug is fixed - continue-on-error: true - uses: zyborg/pester-tests-report@v1.3.2 + uses: zyborg/pester-tests-report@v1 with: test_results_path: test/pester-test-results.xml report_name: pester_tests diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 7ee337a5..6fc65d7f 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -51,7 +51,8 @@ env: TF_INPUT: "false" TF_LOG_PATH: terraform.log TF_LOG: TRACE - TF_VERSION: "0.14.9" # "latest" is supported + # https://github.com/hashicorp/terraform/releases + TF_VERSION: "0.14.9" TF_WORKING_DIR: terraform # Env var concatenation is currently not supported at Workflow or Job scope. See workaround below: From dec2ce29c4a2a1b80d0e6660d889736372dfb3ff Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 10:04:21 +0100 Subject: [PATCH 43/58] Added velero image tag --- terraform/variables.tf | 5 +++++ terraform/velero.tf | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/terraform/variables.tf b/terraform/variables.tf index 60dd2e48..05b17429 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -37,6 +37,11 @@ variable "velero_chart_version" { default = "2.15.0" } +# https://hub.docker.com/r/velero/velero/tags +variable "velero_image_tag" { + default = "v1.5.4" +} + # https://hub.docker.com/r/sonatype/nexus3/tags variable "nexus_image_tag" { default = "3.30.0" diff --git a/terraform/velero.tf b/terraform/velero.tf index 2b1318d8..f0dd9361 100644 --- a/terraform/velero.tf +++ b/terraform/velero.tf @@ -87,9 +87,13 @@ resource "helm_release" "velero" { version = var.velero_chart_version timeout = 600 atomic = true - values = [file("helm/velero_values.yaml")] + set { + name = "image.tag" + value = var.velero_image_tag + } + set { name = "configuration.backupStorageLocation.config.resourceGroup" value = azurerm_resource_group.velero[0].name From b9460a3714fef532a9a541398ba4017d48ad5335 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 10:23:28 +0100 Subject: [PATCH 44/58] Added terraform provider_sensitive_attrs --- terraform/versions.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/versions.tf b/terraform/versions.tf index 1ae92542..4452e32f 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -1,5 +1,8 @@ terraform { + # https://www.terraform.io/docs/language/expressions/references.html#sensitive-resource-attributes + experiments = [provider_sensitive_attrs] + # https://github.com/hashicorp/terraform/releases # https://github.com/hashicorp/terraform/blob/main/CHANGELOG.md required_version = ">= 0.13" From 65172e0f54afe423cb695ab1b673ef5c7ea9fae6 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 10:38:27 +0100 Subject: [PATCH 45/58] Changed argo patch template --- terraform/files/argocd-secret-patch.tmpl.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/files/argocd-secret-patch.tmpl.yaml b/terraform/files/argocd-secret-patch.tmpl.yaml index a341a75f..a58240f2 100644 --- a/terraform/files/argocd-secret-patch.tmpl.yaml +++ b/terraform/files/argocd-secret-patch.tmpl.yaml @@ -1,2 +1,3 @@ data: oidc.azure.clientSecret: ${clientSecretBase64} + test: blah From 6489eb31de5b6be8e35ce73d370556e44eaa0af9 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 10:50:21 +0100 Subject: [PATCH 46/58] Changed argo patch template back --- terraform/argocd_sso.tf | 5 ++--- terraform/files/argocd-secret-patch.tmpl.yaml | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/terraform/argocd_sso.tf b/terraform/argocd_sso.tf index ddbdff9f..25ce7ead 100644 --- a/terraform/argocd_sso.tf +++ b/terraform/argocd_sso.tf @@ -113,6 +113,8 @@ data "template_file" "argocd_secret" { } # https://www.terraform.io/docs/provisioners/local-exec.html +# * uses "experiments = [provider_sensitive_attrs]" to hide output +# https://www.terraform.io/docs/language/expressions/references.html#sensitive-resource-attributes resource "null_resource" "argocd_secret" { triggers = { yaml_contents = filemd5(var.argocd_secret_yaml_path) @@ -125,9 +127,6 @@ resource "null_resource" "argocd_secret" { KUBECONFIG = var.aks_config_path } command = < Date: Fri, 2 Apr 2021 11:04:33 +0100 Subject: [PATCH 47/58] Removed provider_sensitive_attrs --- terraform/versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/versions.tf b/terraform/versions.tf index 4452e32f..c6c7e0c2 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -1,7 +1,7 @@ terraform { # https://www.terraform.io/docs/language/expressions/references.html#sensitive-resource-attributes - experiments = [provider_sensitive_attrs] + # experiments = [provider_sensitive_attrs] # https://github.com/hashicorp/terraform/releases # https://github.com/hashicorp/terraform/blob/main/CHANGELOG.md From df244b153cc35525bc8feac13a6105fba2b00516 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 11:10:43 +0100 Subject: [PATCH 48/58] Added terraform provider_sensitive_attrs --- terraform/versions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/versions.tf b/terraform/versions.tf index c6c7e0c2..4452e32f 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -1,7 +1,7 @@ terraform { # https://www.terraform.io/docs/language/expressions/references.html#sensitive-resource-attributes - # experiments = [provider_sensitive_attrs] + experiments = [provider_sensitive_attrs] # https://github.com/hashicorp/terraform/releases # https://github.com/hashicorp/terraform/blob/main/CHANGELOG.md From 8a5d9240ae08ee6aba6253a2dd4d4201bba12d79 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 11:28:18 +0100 Subject: [PATCH 49/58] Added dev environment --- .github/workflows/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e1deca3..8e122a71 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,6 +75,11 @@ jobs: # view installed software: https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-software runs-on: ubuntu-18.04 + # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idenvironment + environment: + name: dev_environment + url: https://argocd.thehypepipe.co.uk/ + # only run if owner triggered action if: github.actor == github.event.repository.owner.login From ccbdadd34830cb718319f6835f0a8ea6ff001e0f Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 11:37:50 +0100 Subject: [PATCH 50/58] Removed terraform provider_sensitive_attrs --- terraform/versions.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/versions.tf b/terraform/versions.tf index 4452e32f..37fc53cc 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -1,7 +1,8 @@ terraform { # https://www.terraform.io/docs/language/expressions/references.html#sensitive-resource-attributes - experiments = [provider_sensitive_attrs] + # TODO: this currently makes builds fail intermittently + # experiments = [provider_sensitive_attrs] # https://github.com/hashicorp/terraform/releases # https://github.com/hashicorp/terraform/blob/main/CHANGELOG.md From dc719fdc5b9db18b1412b3f2c8f64f55657bd0b3 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 11:40:32 +0100 Subject: [PATCH 51/58] Added environment env var --- .github/workflows/build.yml | 367 +++++++++++++++++------------------- 1 file changed, 177 insertions(+), 190 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e122a71..5c6c365c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -70,7 +70,6 @@ env: jobs: build-and-deploy: - # always pin versions # view installed software: https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-software runs-on: ubuntu-18.04 @@ -78,166 +77,155 @@ jobs: # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idenvironment environment: name: dev_environment - url: https://argocd.thehypepipe.co.uk/ + url: "https://argocd.${{ env.ROOT_DOMAIN_NAME }}" # only run if owner triggered action if: github.actor == github.event.repository.owner.login steps: - - # Checkout - # https://github.com/marketplace/actions/checkout - - uses: actions/checkout@v2 + # Checkout + # https://github.com/marketplace/actions/checkout + - uses: actions/checkout@v2 # specify different branch # NOT required as I've changed the default branch to develop # with: # ref: develop - # Init tasks - inc Env var concatenation - # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files - - name: Init tasks - inc Env var concatenation (Workaround) - run: | - chmod -R +x ./scripts/ - echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV - echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV - echo "ARGOCD_FQDN=argocd.${{ env.ROOT_DOMAIN_NAME }}" >> $GITHUB_ENV - echo "DNS_DOMAIN_NAME=nexus.${{ env.ROOT_DOMAIN_NAME }}" >> $GITHUB_ENV - echo "DOCKER_FQDN=docker.${{ env.ROOT_DOMAIN_NAME }}" >> $GITHUB_ENV - echo "TERRAFORM_STORAGE_ACCOUNT=${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001" >> $GITHUB_ENV - echo "TERRAFORM_STORAGE_RG=${{ env.PREFIX }}-rg-tfstate-dev-001" >> $GITHUB_ENV - echo "VELERO_STORAGE_ACCOUNT=${{ env.PREFIX }}stbckuksouth001" >> $GITHUB_ENV - echo "VELERO_STORAGE_RG=${{ env.PREFIX }}-rg-velero-dev-001" >> $GITHUB_ENV - - - # Login - - name: Login to Azure - run: ./scripts/azure_login.sh - - # This is required when developing after the initial build, and the AKS cluster may have been stopped - # Ensure AKS cluster is running, else timeouts will occur on k8s Terraform apply tasks - - name: Start AKS Cluster - continue-on-error: true - run: ./scripts/start_aks_cluster.sh - - - # Prereqs - - name: Create Storage Account for Terraform state - run: ./scripts/storage_create.sh - - - name: Lookup Storage Key - run: ./scripts/storage_key.sh - - - name: Replace tokens in Terraform config files - run: pwsh -command "./scripts/Replace-Tokens.ps1" - env: - ARGOCD_ADMIN_PASSWORD: ${{ secrets.ARGOCD_ADMIN_PASSWORD }} - HELM_CHART_REPO_DEPLOY_PRIVATE_KEY: ${{ secrets.HELM_CHART_REPO_DEPLOY_PRIVATE_KEY }} - IFTTT_WEBHOOK_KEY: ${{ secrets.IFTTT_WEBHOOK_KEY }} - - - name: Create zip file of Function App - run: pwsh -command "./function_app/CreateFunctionAppZip.ps1" - - - # Terraform - - uses: hashicorp/setup-terraform@v1 - with: - terraform_version: ${{ env.TF_VERSION }} - - - name: Terraform Init / Validate - run: | - terraform init - terraform validate - working-directory: ${{ env.TF_WORKING_DIR }} - - - name: Terraform Lint - run: ./scripts/tflint.sh - env: - TF_WORKING_DIR: ${{ env.TF_WORKING_DIR }} - TFLINT_RULESET_AZURERM_VERSION: ${{ env.TFLINT_RULESET_AZURERM_VERSION }} - TFLINT_VERSION: ${{ env.TFLINT_VERSION }} - - - name: Terraform Plan - id: plan - run: terraform plan -out=${{ env.TF_PLAN }} - working-directory: ${{ env.TF_WORKING_DIR }} - - - name: Terraform Apply - run: ./scripts/terraform_apply.sh - # only apply if changes are present - # https://www.terraform.io/docs/commands/plan.html#detailed-exitcode - # if: steps.plan.outputs.exitcode == 2 - env: - TF_PLAN: ${{ env.TF_PLAN }} - - - # Kubernetes - - name: Deploy kubernetes manifests - run: ./scripts/k8s_manifests_apply.sh - - - name: Wait for resources to be "Ready" - run: ./scripts/wait.sh - - - # Ansible - - name: Run Ansible playbook - run: ./scripts/ansible.sh - env: - NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }} - NEXUS_USER_PASSWORD: ${{ secrets.NEXUS_USER_PASSWORD }} - - - # Docker - - name: Docker repo login - uses: Azure/docker-login@v1 - with: - login-server: ${{ env.DOCKER_FQDN }} - username: ${{ env.NEXUS_USER_USERNAME }} - password: ${{ secrets.NEXUS_USER_PASSWORD }} - - - name: Push images to Docker repo - run: ./scripts/push_docker_images.sh - - - # TODO: Remove once issue has been fixed - # https://github.com/terraform-providers/terraform-provider-azurerm/issues/8546 - - name: Restart Function App - run: az functionapp restart --name "${{ env.PREFIX }}-funcapp" --resource-group "${{ env.PREFIX }}-rg-function-app" - - - # Pester tests - - name: Run Pester tests - continue-on-error: true - run: pwsh -command "./scripts/Start-Test.ps1" - - - name: Archive test artifacts - uses: actions/upload-artifact@v1 - with: - name: test results - path: test/pester-test-results.xml - if: always() - - # remove NuGet proxy repo so pester report step doesnt fail - - name: Unregister NuGet proxy repo - run: pwsh -command "Unregister-PSRepository -Name nuget.org-proxy -Verbose" - - # Shows at the bottom of a run: https://github.com/adamrushuk/devops-lab/runs/1035347513?check_suite_focus=true - - name: Pester report - uses: zyborg/pester-tests-report@v1 - with: - test_results_path: test/pester-test-results.xml - report_name: pester_tests - report_title: Pester Tests - github_token: ${{ secrets.GITHUB_TOKEN }} - - - # Notify - - name: Notify slack - continue-on-error: true - env: - SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - run: ./scripts/send_slack_message.sh "[aks-nexus-velero] Build complete" - + # Init tasks - inc Env var concatenation + # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files + - name: Init tasks - inc Env var concatenation (Workaround) + run: | + chmod -R +x ./scripts/ + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" >> $GITHUB_ENV + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" >> $GITHUB_ENV + echo "ARGOCD_FQDN=argocd.${{ env.ROOT_DOMAIN_NAME }}" >> $GITHUB_ENV + echo "DNS_DOMAIN_NAME=nexus.${{ env.ROOT_DOMAIN_NAME }}" >> $GITHUB_ENV + echo "DOCKER_FQDN=docker.${{ env.ROOT_DOMAIN_NAME }}" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_ACCOUNT=${{ env.PREFIX }}sttfstate${{ env.LOCATION }}001" >> $GITHUB_ENV + echo "TERRAFORM_STORAGE_RG=${{ env.PREFIX }}-rg-tfstate-dev-001" >> $GITHUB_ENV + echo "VELERO_STORAGE_ACCOUNT=${{ env.PREFIX }}stbckuksouth001" >> $GITHUB_ENV + echo "VELERO_STORAGE_RG=${{ env.PREFIX }}-rg-velero-dev-001" >> $GITHUB_ENV + + # Login + - name: Login to Azure + run: ./scripts/azure_login.sh + + # This is required when developing after the initial build, and the AKS cluster may have been stopped + # Ensure AKS cluster is running, else timeouts will occur on k8s Terraform apply tasks + - name: Start AKS Cluster + continue-on-error: true + run: ./scripts/start_aks_cluster.sh + + # Prereqs + - name: Create Storage Account for Terraform state + run: ./scripts/storage_create.sh + + - name: Lookup Storage Key + run: ./scripts/storage_key.sh + + - name: Replace tokens in Terraform config files + run: pwsh -command "./scripts/Replace-Tokens.ps1" + env: + ARGOCD_ADMIN_PASSWORD: ${{ secrets.ARGOCD_ADMIN_PASSWORD }} + HELM_CHART_REPO_DEPLOY_PRIVATE_KEY: ${{ secrets.HELM_CHART_REPO_DEPLOY_PRIVATE_KEY }} + IFTTT_WEBHOOK_KEY: ${{ secrets.IFTTT_WEBHOOK_KEY }} + + - name: Create zip file of Function App + run: pwsh -command "./function_app/CreateFunctionAppZip.ps1" + + # Terraform + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: ${{ env.TF_VERSION }} + + - name: Terraform Init / Validate + run: | + terraform init + terraform validate + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Lint + run: ./scripts/tflint.sh + env: + TF_WORKING_DIR: ${{ env.TF_WORKING_DIR }} + TFLINT_RULESET_AZURERM_VERSION: ${{ env.TFLINT_RULESET_AZURERM_VERSION }} + TFLINT_VERSION: ${{ env.TFLINT_VERSION }} + + - name: Terraform Plan + id: plan + run: terraform plan -out=${{ env.TF_PLAN }} + working-directory: ${{ env.TF_WORKING_DIR }} + + - name: Terraform Apply + run: ./scripts/terraform_apply.sh + # only apply if changes are present + # https://www.terraform.io/docs/commands/plan.html#detailed-exitcode + # if: steps.plan.outputs.exitcode == 2 + env: + TF_PLAN: ${{ env.TF_PLAN }} + + # Kubernetes + - name: Deploy kubernetes manifests + run: ./scripts/k8s_manifests_apply.sh + + - name: Wait for resources to be "Ready" + run: ./scripts/wait.sh + + # Ansible + - name: Run Ansible playbook + run: ./scripts/ansible.sh + env: + NEXUS_ADMIN_PASSWORD: ${{ secrets.NEXUS_ADMIN_PASSWORD }} + NEXUS_USER_PASSWORD: ${{ secrets.NEXUS_USER_PASSWORD }} + + # Docker + - name: Docker repo login + uses: Azure/docker-login@v1 + with: + login-server: ${{ env.DOCKER_FQDN }} + username: ${{ env.NEXUS_USER_USERNAME }} + password: ${{ secrets.NEXUS_USER_PASSWORD }} + + - name: Push images to Docker repo + run: ./scripts/push_docker_images.sh + + # TODO: Remove once issue has been fixed + # https://github.com/terraform-providers/terraform-provider-azurerm/issues/8546 + - name: Restart Function App + run: az functionapp restart --name "${{ env.PREFIX }}-funcapp" --resource-group "${{ env.PREFIX }}-rg-function-app" + + # Pester tests + - name: Run Pester tests + continue-on-error: true + run: pwsh -command "./scripts/Start-Test.ps1" + + - name: Archive test artifacts + uses: actions/upload-artifact@v1 + with: + name: test results + path: test/pester-test-results.xml + if: always() + + # remove NuGet proxy repo so pester report step doesnt fail + - name: Unregister NuGet proxy repo + run: pwsh -command "Unregister-PSRepository -Name nuget.org-proxy -Verbose" + + # Shows at the bottom of a run: https://github.com/adamrushuk/devops-lab/runs/1035347513?check_suite_focus=true + - name: Pester report + uses: zyborg/pester-tests-report@v1 + with: + test_results_path: test/pester-test-results.xml + report_name: pester_tests + report_title: Pester Tests + github_token: ${{ secrets.GITHUB_TOKEN }} + + # Notify + - name: Notify slack + continue-on-error: true + env: + SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + run: ./scripts/send_slack_message.sh "[aks-nexus-velero] Build complete" # used for any windows-only tasks test-windows: @@ -250,42 +238,41 @@ jobs: if: github.actor == github.event.repository.owner.login steps: - # Checkout - - uses: actions/checkout@v2 - - # Init tasks - inc Env var concatenation - # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files - - name: Init tasks - inc Env var concatenation (Workaround) - # * NOTE: different syntax required for Windows agents - run: | - echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" | Out-File -Append -Encoding utf8 -FilePath "$env:GITHUB_ENV" - echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" | Out-File -Append -Encoding utf8 -FilePath "$env:GITHUB_ENV" - echo "DNS_DOMAIN_NAME=nexus.${{ env.ROOT_DOMAIN_NAME }}" | Out-File -Append -Encoding utf8 -FilePath "$env:GITHUB_ENV" - - - # Login - - name: Login to Azure - run: ./scripts/azure_login.ps1 - - # Chocolatey - - name: Test Nexus Chocolatey proxy repo - run: | - choco install velero --source "https://${{ env.DNS_DOMAIN_NAME }}/repository/chocolatey-proxy/" - - # Velero CLI - - name: Test Velero CLI - # NOTE: Some functions cast values to a string to perform comparisons - # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#functions - # ! WARNING: only single quotes work for boolean comparison - if: env.VELERO_ENABLED == 'true' - run: | - az aks get-credentials --resource-group "${{ env.AKS_RG_NAME }}" --name "${{ env.AKS_CLUSTER_NAME }}" --overwrite-existing --admin - - echo "`nVelero version info:" - velero version || true - - echo "`nVelero backup location info:" - velero backup-location get - - echo "`nVelero backup info:" - velero backup get + # Checkout + - uses: actions/checkout@v2 + + # Init tasks - inc Env var concatenation + # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#environment-files + - name: Init tasks - inc Env var concatenation (Workaround) + # * NOTE: different syntax required for Windows agents + run: | + echo "AKS_RG_NAME=${{ env.PREFIX }}-rg-aks-dev-001" | Out-File -Append -Encoding utf8 -FilePath "$env:GITHUB_ENV" + echo "AKS_CLUSTER_NAME=${{ env.PREFIX }}-aks-001" | Out-File -Append -Encoding utf8 -FilePath "$env:GITHUB_ENV" + echo "DNS_DOMAIN_NAME=nexus.${{ env.ROOT_DOMAIN_NAME }}" | Out-File -Append -Encoding utf8 -FilePath "$env:GITHUB_ENV" + + # Login + - name: Login to Azure + run: ./scripts/azure_login.ps1 + + # Chocolatey + - name: Test Nexus Chocolatey proxy repo + run: | + choco install velero --source "https://${{ env.DNS_DOMAIN_NAME }}/repository/chocolatey-proxy/" + + # Velero CLI + - name: Test Velero CLI + # NOTE: Some functions cast values to a string to perform comparisons + # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions#functions + # ! WARNING: only single quotes work for boolean comparison + if: env.VELERO_ENABLED == 'true' + run: | + az aks get-credentials --resource-group "${{ env.AKS_RG_NAME }}" --name "${{ env.AKS_CLUSTER_NAME }}" --overwrite-existing --admin + + echo "`nVelero version info:" + velero version || true + + echo "`nVelero backup location info:" + velero backup-location get + + echo "`nVelero backup info:" + velero backup get From 48f50a28a9d9ce1f293148bb365d8ddb55311e67 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 11:57:54 +0100 Subject: [PATCH 52/58] Added dependency for helm_release.aad_pod_identity --- terraform/akv2k8s_helm.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/akv2k8s_helm.tf b/terraform/akv2k8s_helm.tf index df7e133d..e7e57615 100644 --- a/terraform/akv2k8s_helm.tf +++ b/terraform/akv2k8s_helm.tf @@ -71,4 +71,6 @@ resource "helm_release" "akv2k8s" { name = "controller.logLevel" value = "debug" } + + depends_on = [helm_release.aad_pod_identity] } From 4517ed68ffa2de099ba48f3d8946000402fa2484 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Fri, 2 Apr 2021 14:00:06 +0100 Subject: [PATCH 53/58] Added azure.userAssignedIdentityID for external-dns chart --- terraform/dns.tf | 62 +++++++++++-------- .../files/azureIdentity-external-dns.yaml.tpl | 1 + 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/terraform/dns.tf b/terraform/dns.tf index d2203202..ec9a617d 100644 --- a/terraform/dns.tf +++ b/terraform/dns.tf @@ -43,35 +43,37 @@ resource "kubernetes_namespace" "external_dns" { depends_on = [module.aks] } -data "template_file" "azureIdentity_external_dns" { - template = file(var.azureidentity_external_dns_yaml_path) - vars = { - managedIdentityResourceID = azurerm_user_assigned_identity.external_dns.id - managedIdentityClientID = azurerm_user_assigned_identity.external_dns.client_id - } -} +# TODO: remove if azure.userAssignedIdentityID works in helm_release.external_dns +# data "template_file" "azureIdentity_external_dns" { +# template = file(var.azureidentity_external_dns_yaml_path) +# vars = { +# managedIdentityResourceID = azurerm_user_assigned_identity.external_dns.id +# managedIdentityClientID = azurerm_user_assigned_identity.external_dns.client_id +# } +# } # https://www.terraform.io/docs/provisioners/local-exec.html -resource "null_resource" "azureIdentity_external_dns" { - triggers = { - # always_run = "${timestamp()}" - azureidentity_external_dns_yaml_contents = filemd5(var.azureidentity_external_dns_yaml_path) - } - - provisioner "local-exec" { - interpreter = ["/bin/bash", "-c"] - command = < Date: Fri, 2 Apr 2021 14:37:44 +0100 Subject: [PATCH 54/58] Added env var for KUBECONFIG --- terraform/argocd_helm.tf | 6 ++++-- terraform/gitlab_helm.tf | 6 ++++-- terraform/nexus_helm.tf | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/terraform/argocd_helm.tf b/terraform/argocd_helm.tf index 6995a452..d72f4219 100644 --- a/terraform/argocd_helm.tf +++ b/terraform/argocd_helm.tf @@ -23,8 +23,10 @@ resource "null_resource" "argocd_cert_sync" { provisioner "local-exec" { interpreter = ["/bin/bash", "-c"] - command = <<-EOT - export KUBECONFIG=${var.aks_config_path} + environment = { + KUBECONFIG = var.aks_config_path + } + command = <<-EOT kubectl apply -f ${var.argocd_cert_sync_yaml_path} EOT } diff --git a/terraform/gitlab_helm.tf b/terraform/gitlab_helm.tf index 42174669..8d056642 100644 --- a/terraform/gitlab_helm.tf +++ b/terraform/gitlab_helm.tf @@ -22,8 +22,10 @@ resource "null_resource" "gitlab_cert_sync" { provisioner "local-exec" { interpreter = ["/bin/bash", "-c"] - command = < Date: Fri, 2 Apr 2021 14:38:33 +0100 Subject: [PATCH 55/58] Added aad-pod-identity crds back in --- terraform/dns.tf | 60 +++++++++---------- .../files/azureIdentity-external-dns.yaml.tpl | 1 - 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/terraform/dns.tf b/terraform/dns.tf index ec9a617d..7ccec545 100644 --- a/terraform/dns.tf +++ b/terraform/dns.tf @@ -43,37 +43,37 @@ resource "kubernetes_namespace" "external_dns" { depends_on = [module.aks] } -# TODO: remove if azure.userAssignedIdentityID works in helm_release.external_dns -# data "template_file" "azureIdentity_external_dns" { -# template = file(var.azureidentity_external_dns_yaml_path) -# vars = { -# managedIdentityResourceID = azurerm_user_assigned_identity.external_dns.id -# managedIdentityClientID = azurerm_user_assigned_identity.external_dns.client_id -# } -# } +data "template_file" "azureIdentity_external_dns" { + template = file(var.azureidentity_external_dns_yaml_path) + vars = { + managedIdentityResourceID = azurerm_user_assigned_identity.external_dns.id + managedIdentityClientID = azurerm_user_assigned_identity.external_dns.client_id + } +} # https://www.terraform.io/docs/provisioners/local-exec.html -# TODO: remove if azure.userAssignedIdentityID works in helm_release.external_dns -# resource "null_resource" "azureIdentity_external_dns" { -# triggers = { -# # always_run = "${timestamp()}" -# azureidentity_external_dns_yaml_contents = filemd5(var.azureidentity_external_dns_yaml_path) -# } - -# provisioner "local-exec" { -# interpreter = ["/bin/bash", "-c"] -# command = < Date: Sat, 3 Apr 2021 07:56:13 +0100 Subject: [PATCH 56/58] Bumped aad-pod-identity chart to v4.0.0 --- terraform/aad_pod_identity_helm.tf | 8 +--- terraform/helm/aad_pod_identity_values.yaml | 48 ++++++++++++++++----- terraform/variables.tf | 4 +- 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/terraform/aad_pod_identity_helm.tf b/terraform/aad_pod_identity_helm.tf index 9018b635..8451639b 100644 --- a/terraform/aad_pod_identity_helm.tf +++ b/terraform/aad_pod_identity_helm.tf @@ -51,13 +51,7 @@ resource "helm_release" "aad_pod_identity" { data.template_file.azureIdentities.rendered ] - # should only be required for helm v2 - set { - name = "installCRDs" - value = "false" - } - - # allow Kubenet: https://azure.github.io/aad-pod-identity/docs/configure/aad_pod_identity_on_kubenet/ + # enable if using Kubenet: https://azure.github.io/aad-pod-identity/docs/configure/aad_pod_identity_on_kubenet/ set { name = "nmi.allowNetworkPluginKubenet" value = "false" diff --git a/terraform/helm/aad_pod_identity_values.yaml b/terraform/helm/aad_pod_identity_values.yaml index 31f16b26..153206f1 100644 --- a/terraform/helm/aad_pod_identity_values.yaml +++ b/terraform/helm/aad_pod_identity_values.yaml @@ -1,4 +1,4 @@ -# source: https://github.com/Azure/aad-pod-identity/blob/v1.7.1/charts/aad-pod-identity/values.yaml +# source: https://github.com/Azure/aad-pod-identity/blob/v1.7.5/charts/aad-pod-identity/values.yaml # Default values for aad-pod-identity-helm. # This is a YAML-formatted file. @@ -15,12 +15,6 @@ image: # imagePullSecrets: # - name: myRegistryKeySecretName -# https://github.com/Azure/aad-pod-identity#4-optional-match-pods-in-the-namespace -# By default, AAD Pod Identity matches pods to identities across namespaces. -# To match only pods in the namespace containing AzureIdentity set this to true. -# DEPRECATED - use 'forceNamespaced' instead. -forceNameSpaced: "" - # https://github.com/Azure/aad-pod-identity#4-optional-match-pods-in-the-namespace # By default, AAD Pod Identity matches pods to identities across namespaces. # To match only pods in the namespace containing AzureIdentity set this to true. @@ -51,8 +45,9 @@ operationMode: "standard" mic: image: mic - tag: v1.7.1 + tag: v1.7.5 + # ref: https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/#marking-pod-as-critical priorityClassName: "" # log level. Uses V logs (klog) @@ -69,14 +64,28 @@ mic: podAnnotations: {} + podLabels: {} + ## Node labels for pod assignment ## aad-pod-identity is currently only supported on linux nodeSelector: kubernetes.io/os: linux tolerations: [] + # - key: "CriticalAddonsOnly" + # operator: "Exists" + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity affinity: {} + # nodeAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight 1 + # preference: + # matchExpressions: + # - key: kubernetes.azure.com/mode + # operator: In + # values: + # - system # Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. # ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ @@ -147,8 +156,9 @@ mic: nmi: image: nmi - tag: v1.7.1 + tag: v1.7.5 + # ref: https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/#marking-pod-as-critical priorityClassName: "" # log level. Uses V logs (klog) @@ -165,14 +175,28 @@ nmi: podAnnotations: {} + podLabels: {} + ## Node labels for pod assignment ## aad-pod-identity is currently only supported on linux nodeSelector: kubernetes.io/os: linux tolerations: [] + # - key: "CriticalAddonsOnly" + # operator: "Exists" + # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity affinity: {} + # nodeAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight 1 + # preference: + # matchExpressions: + # - key: kubernetes.azure.com/mode + # operator: In + # values: + # - system # Override iptables update interval in seconds (default is 60) ipTableUpdateTimeIntervalInSeconds: "" @@ -221,6 +245,7 @@ rbac: # NMI requires permissions to get secrets when service principal (type: 1) is used in AzureIdentity. # If using only MSI (type: 0) in AzureIdentity, secret get permission can be disabled by setting this to false. allowAccessToSecrets: true + pspEnabled: false # Create azure identities and bindings # This is a map with the AzureIdentityName being the key and the rest of the blob as value in accordance @@ -247,5 +272,6 @@ azureIdentities: # # The selector will also need to be included in labels for app deployment # selector: "demo" -# If true, install necessary custom resources. -installCRDs: false +# If provided, the userAgent string will be appended to the pod identity user agents for all +# ADAL, ARM and Kube API server requests. +customUserAgent: "" diff --git a/terraform/variables.tf b/terraform/variables.tf index 05b17429..2e89be36 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -65,7 +65,7 @@ variable "akv2k8s_chart_version" { # https://github.com/Azure/aad-pod-identity/blob/master/charts/aad-pod-identity/Chart.yaml#L4 # helm search repo aad-pod-identity/aad-pod-identity variable "aad_pod_identity_chart_version" { - default = "3.0.3" + default = "4.0.0" } # https://bitnami.com/stack/external-dns/helm @@ -78,7 +78,7 @@ variable "external_dns_chart_version" { # https://github.com/weaveworks/kured/tree/master/charts/kured # helm search repo kured/kured variable "kured_chart_version" { - default = "2.4.0" + default = "2.4.1" } # https://github.com/weaveworks/kured#kubernetes--os-compatibility From e197a64f9d8fc111783467502d252f2f964e211e Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 3 Apr 2021 09:17:12 +0100 Subject: [PATCH 57/58] Changed to Azure role-based access control for key_vault --- terraform/akv2k8s_helm.tf | 50 +++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/terraform/akv2k8s_helm.tf b/terraform/akv2k8s_helm.tf index e7e57615..1ff38c6e 100644 --- a/terraform/akv2k8s_helm.tf +++ b/terraform/akv2k8s_helm.tf @@ -10,25 +10,49 @@ data "azurerm_key_vault" "kv" { resource_group_name = var.key_vault_resource_group_name } -resource "azurerm_key_vault_access_policy" "aks" { - key_vault_id = data.azurerm_key_vault.kv.id +# Legacy key vault access policy method +# https://docs.microsoft.com/en-us/azure/key-vault/general/assign-access-policy-portal +# resource "azurerm_key_vault_access_policy" "aks" { +# key_vault_id = data.azurerm_key_vault.kv.id - tenant_id = data.azurerm_subscription.current.tenant_id - object_id = module.aks.kubelet_identity[0].object_id +# tenant_id = data.azurerm_subscription.current.tenant_id +# object_id = module.aks.kubelet_identity[0].object_id - certificate_permissions = [ - "get" - ] +# certificate_permissions = [ +# "get" +# ] - key_permissions = [ - "get" - ] +# key_permissions = [ +# "get" +# ] - secret_permissions = [ - "get" - ] +# secret_permissions = [ +# "get" +# ] +# } + +# Provide key vault access to akv2k8s via Azure role-based access control +# https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment +resource "azurerm_role_assignment" "aks_mi_kv_certs" { + scope = data.azurerm_key_vault.kv.id + role_definition_name = "Key Vault Certificates Officer" + principal_id = module.aks.kubelet_identity[0].object_id + description = "Perform any action on the keys of a key vault, except manage permissions" +} + +resource "azurerm_role_assignment" "aks_mi_kv_keys" { + scope = data.azurerm_key_vault.kv.id + role_definition_name = "Key Vault Crypto User" + principal_id = module.aks.kubelet_identity[0].object_id + description = "Perform cryptographic operations using keys" } +resource "azurerm_role_assignment" "aks_mi_kv_secrets" { + scope = data.azurerm_key_vault.kv.id + role_definition_name = "Key Vault Secrets User" + principal_id = module.aks.kubelet_identity[0].object_id + description = "Provides read-only access to secret contents" +} # Requires "kube_admin_config_raw" as has AAD Auth enabled # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster#kube_admin_config_raw From 64656365a9c09023a7bf9810e8b944db5c7343c0 Mon Sep 17 00:00:00 2001 From: Adam Rush Date: Sat, 3 Apr 2021 09:50:11 +0100 Subject: [PATCH 58/58] Added emojis to important GHA steps --- .github/workflows/build.yml | 6 +++--- .github/workflows/destroy.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c6c365c..a8d9a51e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -151,12 +151,12 @@ jobs: TFLINT_RULESET_AZURERM_VERSION: ${{ env.TFLINT_RULESET_AZURERM_VERSION }} TFLINT_VERSION: ${{ env.TFLINT_VERSION }} - - name: Terraform Plan + - name: ๐Ÿ‘€ Terraform Plan id: plan run: terraform plan -out=${{ env.TF_PLAN }} working-directory: ${{ env.TF_WORKING_DIR }} - - name: Terraform Apply + - name: ๐Ÿš€ Terraform Apply run: ./scripts/terraform_apply.sh # only apply if changes are present # https://www.terraform.io/docs/commands/plan.html#detailed-exitcode @@ -195,7 +195,7 @@ jobs: run: az functionapp restart --name "${{ env.PREFIX }}-funcapp" --resource-group "${{ env.PREFIX }}-rg-function-app" # Pester tests - - name: Run Pester tests + - name: ๐Ÿงช Run Pester tests continue-on-error: true run: pwsh -command "./scripts/Start-Test.ps1" diff --git a/.github/workflows/destroy.yml b/.github/workflows/destroy.yml index 6fc65d7f..cac5aee2 100644 --- a/.github/workflows/destroy.yml +++ b/.github/workflows/destroy.yml @@ -118,7 +118,7 @@ jobs: with: terraform_version: ${{ env.TF_VERSION }} - - name: Terraform destroy + - name: ๐Ÿ’€ Terraform destroy run: | terraform init terraform destroy -no-color -auto-approve