From ccbcf56ec109ea8ce62bf002e6955b20cd5f0fa9 Mon Sep 17 00:00:00 2001 From: David Wobrock Date: Tue, 28 Jan 2025 15:49:35 +0100 Subject: [PATCH] feat: create new hpa template using ContainerResource If Kube version is >=1.3.0, we switch the HPA template to use ContainerResource types instead of Resource. We also add the 'autoscaling.extraMetrics' which shall enable adding scaling for sidecar containers. --- hacks/values/oathkeeper.yaml | 15 +++++++ .../hydra/templates/hpa-controller.yaml | 43 ++++++++++++++++++ helm/charts/hydra/templates/hpa.yaml | 4 +- helm/charts/hydra/values.yaml | 3 ++ .../charts/keto/templates/hpa-controller.yaml | 45 +++++++++++++++++++ helm/charts/keto/templates/hpa.yaml | 4 +- helm/charts/keto/values.yaml | 3 ++ helm/charts/kratos/templates/hpa-kratos.yaml | 40 +++++++++++++++++ helm/charts/kratos/templates/hpa.yaml | 4 +- helm/charts/kratos/values.yaml | 3 ++ .../oathkeeper/templates/hpa-controller.yaml | 4 +- helm/charts/oathkeeper/templates/hpa.yaml | 43 ++++++++++++++++++ helm/charts/oathkeeper/values.yaml | 3 ++ 13 files changed, 210 insertions(+), 4 deletions(-) create mode 100644 helm/charts/hydra/templates/hpa-controller.yaml create mode 100644 helm/charts/keto/templates/hpa-controller.yaml create mode 100644 helm/charts/kratos/templates/hpa-kratos.yaml create mode 100644 helm/charts/oathkeeper/templates/hpa.yaml diff --git a/hacks/values/oathkeeper.yaml b/hacks/values/oathkeeper.yaml index 5cb86d55be..29ea896d7a 100644 --- a/hacks/values/oathkeeper.yaml +++ b/hacks/values/oathkeeper.yaml @@ -63,6 +63,21 @@ deployment: behavior: scaleDown: stabilizationWindowSeconds: 60 + extraMetrics: + - type: ContainerResource + containerResource: + name: cpu + container: sidecar + target: + type: Utilization + averageUtilization: 80 + - type: ContainerResource + containerResource: + name: memory + container: sidecar + target: + type: Utilization + averageUtilization: 80 strategy: type: RollingUpdate rollingUpdate: diff --git a/helm/charts/hydra/templates/hpa-controller.yaml b/helm/charts/hydra/templates/hpa-controller.yaml new file mode 100644 index 0000000000..cae42a23b3 --- /dev/null +++ b/helm/charts/hydra/templates/hpa-controller.yaml @@ -0,0 +1,43 @@ +{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}} +{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}} +{{- if and .Values.deployment.autoscaling.enabled $containerResourceAvailable }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + {{- if .Release.Namespace }} + namespace: {{ .Release.Namespace }} + {{- end }} + name: {{ include "hydra.fullname" . }} + labels: + {{- include "hydra.labels" . | nindent 4 }} +spec: + {{- with .Values.deployment.autoscaling.behavior }} + behavior: {{- toYaml . | nindent 4 }} + {{- end }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "hydra.fullname" . }} + minReplicas: {{ .Values.deployment.autoscaling.minReplicas }} + maxReplicas: {{ .Values.deployment.autoscaling.maxReplicas }} + metrics: + {{- with .Values.deployment.autoscaling.targetMemory }} + - type: ContainerResource + containerResource: + name: memory + container: {{ $.Chart.Name }} + target: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.deployment.autoscaling.targetCPU}} + - type: ContainerResource + containerResource: + name: cpu + container: {{ $.Chart.Name }} + target: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not (empty .Values.deployment.autoscaling.extraMetrics) }} + {{- toYaml .Values.deployment.autoscaling.extraMetrics | nindent 2 }} + {{- end }} +{{- end }} diff --git a/helm/charts/hydra/templates/hpa.yaml b/helm/charts/hydra/templates/hpa.yaml index 5c4f5df1b3..814472378f 100644 --- a/helm/charts/hydra/templates/hpa.yaml +++ b/helm/charts/hydra/templates/hpa.yaml @@ -1,4 +1,6 @@ -{{- if .Values.deployment.autoscaling.enabled }} +{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}} +{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}} +{{- if and .Values.deployment.autoscaling.enabled (not $containerResourceAvailable) }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/charts/hydra/values.yaml b/helm/charts/hydra/values.yaml index f81ff6017b..d37d27ad13 100644 --- a/helm/charts/hydra/values.yaml +++ b/helm/charts/hydra/values.yaml @@ -334,6 +334,9 @@ deployment: # -- Set custom behavior # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior behavior: {} + # -- Add extraContainer container resource metrics + # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics + extraMetrics: [] # -- Default probe timers readinessProbe: diff --git a/helm/charts/keto/templates/hpa-controller.yaml b/helm/charts/keto/templates/hpa-controller.yaml new file mode 100644 index 0000000000..97c86b827f --- /dev/null +++ b/helm/charts/keto/templates/hpa-controller.yaml @@ -0,0 +1,45 @@ +{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}} +{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}} +{{- $autoscaling := ternary .Values.deployment.autoscaling .Values.autoscaling (not (empty .Values.deployment.autoscaling )) -}} + +{{- if and $autoscaling.enabled $containerResourceAvailable }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "keto.fullname" . }} + {{- if .Release.Namespace }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: + {{- include "keto.labels" . | nindent 4 }} +spec: + {{- with $autoscaling.behavior }} + behavior: {{- toYaml . | nindent 4 }} + {{- end }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "keto.fullname" . }} + minReplicas: {{ $autoscaling.minReplicas }} + maxReplicas: {{ $autoscaling.maxReplicas }} + metrics: + {{- with $autoscaling.targetMemory }} + - type: ContainerResource + containerResource: + name: memory + container: {{ $.Chart.Name }} + target: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $autoscaling.targetCPU}} + - type: ContainerResource + containerResource: + name: cpu + container: {{ $.Chart.Name }} + target: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not (empty $autoscaling.extraMetrics) }} + {{- toYaml $autoscaling.extraMetrics | nindent 2 }} + {{- end }} +{{- end }} diff --git a/helm/charts/keto/templates/hpa.yaml b/helm/charts/keto/templates/hpa.yaml index acf1a59f5a..d391c77398 100644 --- a/helm/charts/keto/templates/hpa.yaml +++ b/helm/charts/keto/templates/hpa.yaml @@ -1,6 +1,8 @@ +{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}} +{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}} {{- $autoscaling := ternary .Values.deployment.autoscaling .Values.autoscaling (not (empty .Values.deployment.autoscaling )) -}} -{{- if $autoscaling.enabled }} +{{- if and $autoscaling.enabled (not $containerResourceAvailable) }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/charts/keto/values.yaml b/helm/charts/keto/values.yaml index 0d24fea653..965bdbdb95 100644 --- a/helm/charts/keto/values.yaml +++ b/helm/charts/keto/values.yaml @@ -347,6 +347,9 @@ deployment: # -- Set custom behavior # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior behavior: {} + # -- Add extraContainer container resource metrics + # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics + extraMetrics: [] nodeSelector: {} diff --git a/helm/charts/kratos/templates/hpa-kratos.yaml b/helm/charts/kratos/templates/hpa-kratos.yaml new file mode 100644 index 0000000000..b2cd850a60 --- /dev/null +++ b/helm/charts/kratos/templates/hpa-kratos.yaml @@ -0,0 +1,40 @@ +{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}} +{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}} +{{- if and .Values.autoscaling.enabled $containerResourceAvailable }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "kratos.fullname" . }} + labels: + {{- include "kratos.labels" . | nindent 4 }} +spec: + {{- with .Values.autoscaling.behavior }} + behavior: {{- toYaml . | nindent 4 }} + {{- end }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "kratos.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- with .Values.autoscaling.targetMemory }} + - type: ContainerResource + containerResource: + name: memory + container: {{ $.Chart.Name }} + target: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.autoscaling.targetCPU}} + - type: ContainerResource + containerResource: + name: cpu + container: {{ $.Chart.Name }} + target: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not (empty $.Values.autoscaling.extraMetrics) }} + {{- toYaml .Values.autoscaling.extraMetrics | nindent 2 }} + {{- end }} +{{- end }} diff --git a/helm/charts/kratos/templates/hpa.yaml b/helm/charts/kratos/templates/hpa.yaml index aa8df05619..d844f9a3b0 100644 --- a/helm/charts/kratos/templates/hpa.yaml +++ b/helm/charts/kratos/templates/hpa.yaml @@ -1,4 +1,6 @@ -{{- if .Values.autoscaling.enabled }} +{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}} +{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}} +{{- if and .Values.autoscaling.enabled (not $containerResourceAvailable) }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/charts/kratos/values.yaml b/helm/charts/kratos/values.yaml index 316fd16ae8..1a0f8e75b2 100644 --- a/helm/charts/kratos/values.yaml +++ b/helm/charts/kratos/values.yaml @@ -531,6 +531,9 @@ autoscaling: # -- Set custom behavior # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior behavior: {} + # -- Add extraContainer container resource metrics + # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics + extraMetrics: [] ## -- Values for initialization job job: diff --git a/helm/charts/oathkeeper/templates/hpa-controller.yaml b/helm/charts/oathkeeper/templates/hpa-controller.yaml index b80f4a2ae1..fb131475ce 100644 --- a/helm/charts/oathkeeper/templates/hpa-controller.yaml +++ b/helm/charts/oathkeeper/templates/hpa-controller.yaml @@ -1,4 +1,6 @@ -{{- if .Values.deployment.autoscaling.enabled }} +{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}} +{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}} +{{- if and .Values.deployment.autoscaling.enabled (not $containerResourceAvailable) }} apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: diff --git a/helm/charts/oathkeeper/templates/hpa.yaml b/helm/charts/oathkeeper/templates/hpa.yaml new file mode 100644 index 0000000000..df931f83f3 --- /dev/null +++ b/helm/charts/oathkeeper/templates/hpa.yaml @@ -0,0 +1,43 @@ +{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}} +{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}} +{{- if and .Values.deployment.autoscaling.enabled $containerResourceAvailable }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "oathkeeper.fullname" . }} + {{- if .Release.Namespace }} + namespace: {{ .Release.Namespace }} + {{- end }} + labels: + {{- include "oathkeeper.labels" . | nindent 4 }} +spec: + {{- with .Values.deployment.autoscaling.behavior }} + behavior: {{- toYaml . | nindent 4 }} + {{- end }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "oathkeeper.fullname" . }} + minReplicas: {{ .Values.deployment.autoscaling.minReplicas }} + maxReplicas: {{ .Values.deployment.autoscaling.maxReplicas }} + metrics: + {{- with .Values.deployment.autoscaling.targetMemory }} + - type: ContainerResource + containerResource: + name: memory + container: {{ $.Chart.Name }} + target: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.deployment.autoscaling.targetCPU}} + - type: ContainerResource + containerResource: + name: cpu + container: {{ $.Chart.Name }} + target: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if not (empty $.Values.deployment.autoscaling.extraMetrics) }} + {{- toYaml .Values.deployment.autoscaling.extraMetrics | nindent 2 }} + {{- end }} +{{- end }} diff --git a/helm/charts/oathkeeper/values.yaml b/helm/charts/oathkeeper/values.yaml index 333f14ba23..13bb42f643 100644 --- a/helm/charts/oathkeeper/values.yaml +++ b/helm/charts/oathkeeper/values.yaml @@ -374,6 +374,9 @@ deployment: # -- Set custom behavior # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior behavior: {} + # -- Add extraContainer container resource metrics + # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics + extraMetrics: [] # -- Configure node affinity affinity: {}