From aba5237b281af58a7950deb48bcec1e3f3749498 Mon Sep 17 00:00:00 2001 From: Hayden Spitzley Date: Fri, 28 Feb 2025 07:56:16 -0700 Subject: [PATCH 1/5] feat: add support for k8s jobs to stack helm chart --- Makefile | 2 + common.mk | 8 ++ stack/Makefile | 2 + stack/templates/job.yaml | 165 ++++++++++++++++++++++++++++++++++ stack/tests/cronjob_test.yaml | 2 +- stack/tests/job_test.yaml | 122 +++++++++++++++++++++++++ stack/values.yaml | 49 ++++++++++ 7 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 Makefile create mode 100755 common.mk create mode 100644 stack/Makefile create mode 100644 stack/templates/job.yaml create mode 100644 stack/tests/job_test.yaml diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..12c496b --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ + +include ./common.mk diff --git a/common.mk b/common.mk new file mode 100755 index 0000000..6e1942a --- /dev/null +++ b/common.mk @@ -0,0 +1,8 @@ + +.PHONY: test +test: + @for i in $$(find . -type d); do \ + if [ -e "$$i/Chart.yaml" ]; then \ + helm unittest $$i; \ + fi; \ + done diff --git a/stack/Makefile b/stack/Makefile new file mode 100644 index 0000000..5fe6707 --- /dev/null +++ b/stack/Makefile @@ -0,0 +1,2 @@ + +include ../common.mk diff --git a/stack/templates/job.yaml b/stack/templates/job.yaml new file mode 100644 index 0000000..4b4ec33 --- /dev/null +++ b/stack/templates/job.yaml @@ -0,0 +1,165 @@ +{{ $global := . }} +{{ range $jobName, $jobValues := .Values.jobs }} + {{- $globalValuesDict := $global.Values.global | toYaml -}} + {{- $values := fromYaml $globalValuesDict -}} + {{- $values = set $values "name" $jobName -}} + {{- $values := mergeOverwrite $values $jobValues -}} + {{- $job := dict "Chart" $global.Chart "Release" $global.Release "Capabilities" $global.Capabilities "Values" $values -}} +{{- with $job -}} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "service.fullname" . }} +spec: + {{- if .Values.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.activeDeadlineSeconds }} + {{- end }} + {{- if .Values.backoffLimit }} + backoffLimit: {{ .Values.backoffLimit }} + {{- end }} + {{- if .Values.completions }} + completions: {{ .Values.completions }} + {{- end }} + {{- if .Values.parallelism }} + parallelism: {{ .Values.parallelism }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "service.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "service.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + shareProcessNamespace: {{ .Values.shareProcessNamespace }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if and .Values.args (ne (len .Values.args) 0) }} + args: + {{- toYaml .Values.args | nindent 12 }} + {{- end }} + {{- if and .Values.command (ne (len .Values.command) 0) }} + command: + {{- toYaml .Values.command | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + {{- include "container.probe" .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- include "container.probe" .Values.readinessProbe | nindent 12 }} + {{- if eq .Values.startupProbe.enabled true }} + startupProbe: + {{- include "container.probe" (omit .Values.startupProbe "enabled") | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if or (and .Values.persistence.enabled .Values.persistence.mountPath) .Values.volumeMounts}} + volumeMounts: + {{- if and .Values.persistence.enabled .Values.persistence.mountPath }} + - name: data + mountPath: {{ .Values.persistence.mountPath }} + {{- end }} + {{- if .Values.volumeMounts }} + {{- toYaml .Values.volumeMounts | nindent 12 }} + {{- end }} + {{- end }} + {{- include "service.configuration" . | nindent 10}} + {{- include "service.nonsensitiveEnvVars" (list $global.Values.global .Values) | nindent 10 }} + {{- range $i, $container := .Values.sidecars }} + {{- $imageDict := fromYaml (include "image" $container) }} + {{- $container = mergeOverwrite $container $imageDict }} + {{- with omit $container "envFrom" "env" }} + - {{- toYaml . | nindent 10 }} + {{- include "service.configuration" $job | nindent 10}} + {{- end }} + {{- include "service.nonsensitiveEnvVars" (list $global.Values.global $job.Values $container) | nindent 10 }} + {{- end }} + initContainers: + {{- range $i, $container := .Values.initContainers }} + {{- $imageDict := fromYaml (include "image" $container) }} + {{- $container = mergeOverwrite $container $imageDict }} + {{- with omit $container "envFrom" "env" }} + - {{- toYaml . | nindent 10 }} + {{- include "service.configuration" $job | nindent 10}} + {{- end }} + {{- include "service.nonsensitiveEnvVars" (list $global.Values.global $job.Values $container) | nindent 10 }} + {{- end }} + dnsPolicy: {{ .Values.dnsPolicy }} + restartPolicy: {{ .Values.restartPolicy }} + {{- if or .Values.persistence.enabled .Values.volumes}} + volumes: + {{- if .Values.persistence.enabled }} + - name: data + persistentVolumeClaim: + claimName: {{ include "service.claimName" . }} + {{- end }} + {{- if .Values.volumes }} + {{- toYaml .Values.volumes | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.topologySpreadConstraints }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- else }} + topologySpreadConstraints: + - maxSkew: 2 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + {{- include "service.selectorLabels" . | nindent 14 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +{{ if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "service.serviceAccountName" . }} + labels: + {{- include "service.labels" . | nindent 4 }} + {{- with mergeOverwrite + (dict) + .Values.annotations + .Values.serviceAccount.annotations + }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +--- +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/stack/tests/cronjob_test.yaml b/stack/tests/cronjob_test.yaml index 3db6aca..7a17a28 100644 --- a/stack/tests/cronjob_test.yaml +++ b/stack/tests/cronjob_test.yaml @@ -110,6 +110,6 @@ tests: - containsDocument: apiVersion: v1 kind: ServiceAccount - name: "release-name-stack-job1" + name: "release-name-stack-job2" not: true diff --git a/stack/tests/job_test.yaml b/stack/tests/job_test.yaml new file mode 100644 index 0000000..245c210 --- /dev/null +++ b/stack/tests/job_test.yaml @@ -0,0 +1,122 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json +suite: test jobs +templates: + - job.yaml +tests: + - it: should work + set: + jobs: + job1: + activeDeadlineSeconds: 300 + backoffLimit: 2 + completions: 5 + parallelism: 3 + serviceAccount: + create: true + annotations: + "eks.amazonaws.com/role-arn": some-role + image: + repository: my-repo + tag: sha-mytag + command: ["hello-world"] + args: ["arg1", "arg2"] + asserts: + - hasDocuments: + count: 2 + - documentIndex: 0 + containsDocument: + apiVersion: batch/v1 + kind: Job + name: "release-name-stack-job1" + template: job.yaml + - documentIndex: 0 + equal: + path: metadata.name + value: "release-name-stack-job1" + - documentIndex: 0 + equal: + path: spec.template.spec.containers[0].image + value: "my-repo:sha-mytag" + - documentIndex: 0 + equal: + path: spec.template.spec.containers[0].command + value: ["hello-world"] + - documentIndex: 0 + equal: + path: spec.template.spec.containers[0].args + value: ["arg1", "arg2"] + - documentIndex: 0 + equal: + path: spec.activeDeadlineSeconds + value: 300 + - documentIndex: 0 + equal: + path: spec.backoffLimit + value: 2 + - documentIndex: 0 + equal: + path: spec.completions + value: 5 + - documentIndex: 0 + equal: + path: spec.parallelism + value: 3 + - documentIndex: 1 + containsDocument: + apiVersion: v1 + kind: ServiceAccount + name: "release-name-stack-job1" + - documentIndex: 1 + equal: + path: metadata.name + value: "release-name-stack-job1" + - documentIndex: 1 + equal: + path: metadata.annotations["eks.amazonaws.com/role-arn"] + value: "some-role" + - documentIndex: 1 + equal: + path: metadata.labels["app.kubernetes.io/name"] + value: "stack" + - documentIndex: 1 + equal: + path: metadata.labels["app.kubernetes.io/instance"] + value: "RELEASE-NAME" + - documentIndex: 1 + equal: + path: metadata.labels["app.kubernetes.io/version"] + value: "1.16.0" + - documentIndex: 1 + equal: + path: metadata.labels["app.kubernetes.io/managed-by"] + value: "Helm" + - documentIndex: 1 + equal: + path: metadata.labels["app.kubernetes.io/service"] + value: "release-name-stack-job1" + - documentIndex: 1 + equal: + path: automountServiceAccountToken + value: true + - it: should not make service account for job + set: + jobs: + job2: + image: + repository: my-repo + tag: sha-mytag + command: ["hello-world"] + args: ["arg1", "arg2"] + asserts: + - hasDocuments: + count: 1 + - containsDocument: + apiVersion: batch/v1 + kind: Job + name: "release-name-stack-job2" + template: job.yaml + - containsDocument: + apiVersion: v1 + kind: ServiceAccount + name: "release-name-stack-job2" + not: true diff --git a/stack/values.yaml b/stack/values.yaml index 1780dcc..d3d6911 100644 --- a/stack/values.yaml +++ b/stack/values.yaml @@ -390,3 +390,52 @@ cronJobs: # tag: "latest" # command: ["command1", "command2"] # args: ["arg1", "arg2"] + +## @param jobs Jobs to deploy, all values in the above global section are inherited by the jobs and each job can override them +jobs: + # {} + # job1: + # activeDeadlineSeconds: 300 + # backoffLimit: 2 + # completions: 5 + # parallelism: 3 + # serviceAccount: + # create: false + # image: + # repository: my-repo + # tag: sha-mytag + # command: ["hello-world"] + # args: ["arg1", "arg2"] + job1: + activeDeadlineSeconds: 300 + backoffLimit: 2 + completions: 5 + parallelism: 3 + serviceAccount: + create: true + annotations: + "eks.amazonaws.com/role-arn": some-role + image: + repository: my-repo + tag: sha-mytag + command: ["hello-world"] + args: ["arg1", "arg2"] + # job1: + # serviceAccount: + # create: true + # image: + # repository: nginx + # pullPolicy: IfNotPresent + # tag: "latest" + # command: ["command1", "command2"] + # args: ["arg1", "arg2"] + # concurrencyPolicy: Forbid + # schedule: "* * * * *" + # replicaCount: 2 + # sidecars: + # - name: sidecar1 + # image: "sidecar1:latest" + # initContainers: + # - name: initContainer1 + # image: "alpine:latest" + # command: ["echo", "Hello World"] From 5168bd096f3b49adc22420a674099062010ed6b8 Mon Sep 17 00:00:00 2001 From: Hayden Spitzley Date: Fri, 28 Feb 2025 08:00:39 -0700 Subject: [PATCH 2/5] values --- stack/values.yaml | 39 ++++----------------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/stack/values.yaml b/stack/values.yaml index d3d6911..bf38ac1 100644 --- a/stack/values.yaml +++ b/stack/values.yaml @@ -393,49 +393,18 @@ cronJobs: ## @param jobs Jobs to deploy, all values in the above global section are inherited by the jobs and each job can override them jobs: - # {} + {} # job1: # activeDeadlineSeconds: 300 # backoffLimit: 2 # completions: 5 # parallelism: 3 # serviceAccount: - # create: false + # create: true + # annotations: + # "eks.amazonaws.com/role-arn": some-role # image: # repository: my-repo # tag: sha-mytag # command: ["hello-world"] # args: ["arg1", "arg2"] - job1: - activeDeadlineSeconds: 300 - backoffLimit: 2 - completions: 5 - parallelism: 3 - serviceAccount: - create: true - annotations: - "eks.amazonaws.com/role-arn": some-role - image: - repository: my-repo - tag: sha-mytag - command: ["hello-world"] - args: ["arg1", "arg2"] - # job1: - # serviceAccount: - # create: true - # image: - # repository: nginx - # pullPolicy: IfNotPresent - # tag: "latest" - # command: ["command1", "command2"] - # args: ["arg1", "arg2"] - # concurrencyPolicy: Forbid - # schedule: "* * * * *" - # replicaCount: 2 - # sidecars: - # - name: sidecar1 - # image: "sidecar1:latest" - # initContainers: - # - name: initContainer1 - # image: "alpine:latest" - # command: ["echo", "Hello World"] From 7676751d2fce8e9eccbc831c71178c2eea501fad Mon Sep 17 00:00:00 2001 From: hspitzley-czi Date: Fri, 28 Feb 2025 15:01:02 +0000 Subject: [PATCH 3/5] chore: Update README/Schema for changed charts --- stack/README.md | 1 + stack/values.schema.json | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/stack/README.md b/stack/README.md index fe755b3..b25bcc3 100644 --- a/stack/README.md +++ b/stack/README.md @@ -149,3 +149,4 @@ A Helm chart for deploying an Argus stack. | `global.oidcProxy.resources.requests.memory` | Memory request | `4Gi` | | `services` | Services to deploy, all values in the above global section are inherited by the services and each service can override them | `{}` | | `cronJobs` | Cron jobs to deploy, all values in the above global section are inherited by the cron jobs and each cron job can override them | `{}` | +| `jobs` | Jobs to deploy, all values in the above global section are inherited by the jobs and each job can override them | `{}` | diff --git a/stack/values.schema.json b/stack/values.schema.json index ae797ed..d2ab317 100644 --- a/stack/values.schema.json +++ b/stack/values.schema.json @@ -648,6 +648,11 @@ "type": "object", "description": "Cron jobs to deploy, all values in the above global section are inherited by the cron jobs and each cron job can override them", "default": {} + }, + "jobs": { + "type": "object", + "description": "Jobs to deploy, all values in the above global section are inherited by the jobs and each job can override them", + "default": {} } } } \ No newline at end of file From 422c450bf3e9a028706c716cce52cb6be171138a Mon Sep 17 00:00:00 2001 From: Hayden Spitzley Date: Fri, 28 Feb 2025 14:22:13 -0700 Subject: [PATCH 4/5] tweaks --- stack/templates/job.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/stack/templates/job.yaml b/stack/templates/job.yaml index 4b4ec33..d0b93dd 100644 --- a/stack/templates/job.yaml +++ b/stack/templates/job.yaml @@ -11,10 +11,20 @@ apiVersion: batch/v1 kind: Job metadata: name: {{ include "service.fullname" . }} + labels: + {{- include "service.labels" . | nindent 4 }} + annotations: + {{- toYaml (mergeOverwrite + (dict) + (fromYaml (include "stack.annotations" $job )) + ) | nindent 4 }} spec: {{- if .Values.activeDeadlineSeconds }} activeDeadlineSeconds: {{ .Values.activeDeadlineSeconds }} {{- end }} + {{- if .Values.ttlSecondsAfterFinished }} + ttlSecondsAfterFinished: {{ .Values.ttlSecondsAfterFinished }} + {{- end }} {{- if .Values.backoffLimit }} backoffLimit: {{ .Values.backoffLimit }} {{- end }} @@ -104,7 +114,11 @@ spec: {{- include "service.nonsensitiveEnvVars" (list $global.Values.global $job.Values $container) | nindent 10 }} {{- end }} dnsPolicy: {{ .Values.dnsPolicy }} + {{- if eq .Values.restartPolicy "Always" }} + restartPolicy: OnFailure + {{- else }} restartPolicy: {{ .Values.restartPolicy }} + {{- end }} {{- if or .Values.persistence.enabled .Values.volumes}} volumes: {{- if .Values.persistence.enabled }} From 16472f25b3a6e3e57c8ca46bae6adf2b534933e5 Mon Sep 17 00:00:00 2001 From: Hayden Spitzley Date: Mon, 3 Mar 2025 08:33:15 -0700 Subject: [PATCH 5/5] disable linkerd --- stack/templates/job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack/templates/job.yaml b/stack/templates/job.yaml index d0b93dd..877d1d8 100644 --- a/stack/templates/job.yaml +++ b/stack/templates/job.yaml @@ -36,7 +36,7 @@ spec: {{- end }} template: metadata: - {{- with .Values.podAnnotations }} + {{- with mergeOverwrite .Values.podAnnotations (dict "linkerd.io/inject" "disabled") }} annotations: {{- toYaml . | nindent 8 }} {{- end }}