feat: create new hpa template using ContainerResource

If Kube version is >=1.3.0, we switch the HPA template to use ContainerResource types instead of Resource. We also add the 'autoscaling.extraMetrics' which shall enable adding scaling for sidecar containers.
ory · Jan 28, 2025 · ccbcf56 · ccbcf56
1 parent f4ada24
commit ccbcf56
Show file tree

Hide file tree

Showing 13 changed files with 210 additions and 4 deletions.
diff --git a/hacks/values/oathkeeper.yaml b/hacks/values/oathkeeper.yaml
@@ -63,6 +63,21 @@ deployment:
     behavior:
       scaleDown:
         stabilizationWindowSeconds: 60
+    extraMetrics:
+      - type: ContainerResource
+        containerResource:
+          name: cpu
+          container: sidecar
+          target:
+            type: Utilization
+            averageUtilization: 80
+      - type: ContainerResource
+        containerResource:
+          name: memory
+          container: sidecar
+          target:
+            type: Utilization
+            averageUtilization: 80
   strategy:
     type: RollingUpdate
     rollingUpdate:

diff --git a/helm/charts/hydra/templates/hpa-controller.yaml b/helm/charts/hydra/templates/hpa-controller.yaml
@@ -0,0 +1,43 @@
+{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}}
+{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}}
+{{- if and .Values.deployment.autoscaling.enabled $containerResourceAvailable }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  {{- if .Release.Namespace }}
+  namespace: {{ .Release.Namespace }}
+  {{- end }}
+  name: {{ include "hydra.fullname" . }}
+  labels:
+    {{- include "hydra.labels" . | nindent 4 }}
+spec:
+  {{- with .Values.deployment.autoscaling.behavior }}
+  behavior: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "hydra.fullname" . }}
+  minReplicas: {{ .Values.deployment.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.deployment.autoscaling.maxReplicas }}
+  metrics:
+  {{- with .Values.deployment.autoscaling.targetMemory }}
+  - type: ContainerResource
+    containerResource:
+      name: memory
+      container: {{ $.Chart.Name }}
+      target:
+        {{- toYaml . | nindent 8 }}
+  {{- end }}
+  {{- with .Values.deployment.autoscaling.targetCPU}}
+  - type: ContainerResource
+    containerResource:
+      name: cpu
+      container: {{ $.Chart.Name }}
+      target:
+        {{- toYaml . | nindent 8 }}
+  {{- end }}
+  {{- if not (empty .Values.deployment.autoscaling.extraMetrics) }}
+  {{- toYaml .Values.deployment.autoscaling.extraMetrics | nindent 2 }}
+  {{- end }}
+{{- end }}
diff --git a/helm/charts/hydra/templates/hpa.yaml b/helm/charts/hydra/templates/hpa.yaml
@@ -1,4 +1,6 @@
-{{- if .Values.deployment.autoscaling.enabled }}
+{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}}
+{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}}
+{{- if and .Values.deployment.autoscaling.enabled (not $containerResourceAvailable) }}
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:

diff --git a/helm/charts/hydra/values.yaml b/helm/charts/hydra/values.yaml
@@ -334,6 +334,9 @@ deployment:
     # -- Set custom behavior
     # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior
     behavior: {}
+    # -- Add extraContainer container resource metrics
+    # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics
+    extraMetrics: []
 
   # -- Default probe timers
   readinessProbe:

diff --git a/helm/charts/keto/templates/hpa-controller.yaml b/helm/charts/keto/templates/hpa-controller.yaml
@@ -0,0 +1,45 @@
+{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}}
+{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}}
+{{- $autoscaling := ternary .Values.deployment.autoscaling .Values.autoscaling (not (empty .Values.deployment.autoscaling )) -}}
+
+{{- if and $autoscaling.enabled $containerResourceAvailable }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "keto.fullname" . }}
+  {{- if .Release.Namespace }}
+  namespace: {{ .Release.Namespace }}
+  {{- end }}
+  labels:
+    {{- include "keto.labels" . | nindent 4 }}
+spec:
+  {{- with $autoscaling.behavior }}
+  behavior: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "keto.fullname" . }}
+  minReplicas: {{ $autoscaling.minReplicas }}
+  maxReplicas: {{ $autoscaling.maxReplicas }}
+  metrics:
+  {{- with $autoscaling.targetMemory }}
+  - type: ContainerResource
+    containerResource:
+      name: memory
+      container: {{ $.Chart.Name }}
+      target:
+        {{- toYaml . | nindent 8 }}
+  {{- end }}
+  {{- with $autoscaling.targetCPU}}
+  - type: ContainerResource
+    containerResource:
+      name: cpu
+      container: {{ $.Chart.Name }}
+      target:
+        {{- toYaml . | nindent 8 }}
+  {{- end }}
+  {{- if not (empty $autoscaling.extraMetrics) }}
+  {{- toYaml $autoscaling.extraMetrics | nindent 2 }}
+  {{- end }}
+{{- end }}
diff --git a/helm/charts/keto/templates/hpa.yaml b/helm/charts/keto/templates/hpa.yaml
@@ -1,6 +1,8 @@
+{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}}
+{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}}
 {{- $autoscaling := ternary .Values.deployment.autoscaling .Values.autoscaling (not (empty .Values.deployment.autoscaling )) -}}
 
-{{- if $autoscaling.enabled }}
+{{- if and $autoscaling.enabled (not $containerResourceAvailable) }}
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:

diff --git a/helm/charts/keto/values.yaml b/helm/charts/keto/values.yaml
@@ -347,6 +347,9 @@ deployment:
     # -- Set custom behavior
     # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior
     behavior: {}
+    # -- Add extraContainer container resource metrics
+    # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics
+    extraMetrics: []
 
   nodeSelector: {}
 

diff --git a/helm/charts/kratos/templates/hpa-kratos.yaml b/helm/charts/kratos/templates/hpa-kratos.yaml
@@ -0,0 +1,40 @@
+{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}}
+{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}}
+{{- if and .Values.autoscaling.enabled $containerResourceAvailable }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "kratos.fullname" . }}
+  labels:
+    {{- include "kratos.labels" . | nindent 4 }}
+spec:
+  {{- with .Values.autoscaling.behavior }}
+  behavior: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "kratos.fullname" . }}
+  minReplicas: {{ .Values.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  metrics:
+  {{- with .Values.autoscaling.targetMemory }}
+  - type: ContainerResource
+    containerResource:
+      name: memory
+      container: {{ $.Chart.Name }}
+      target:
+        {{- toYaml . | nindent 8 }}
+  {{- end }}
+  {{- with .Values.autoscaling.targetCPU}}
+  - type: ContainerResource
+    containerResource:
+      name: cpu
+      container: {{ $.Chart.Name }}
+      target:
+        {{- toYaml . | nindent 8 }}
+  {{- end }}
+  {{- if not (empty $.Values.autoscaling.extraMetrics) }}
+  {{- toYaml .Values.autoscaling.extraMetrics | nindent 2 }}
+  {{- end }}
+{{- end }}
diff --git a/helm/charts/kratos/templates/hpa.yaml b/helm/charts/kratos/templates/hpa.yaml
@@ -1,4 +1,6 @@
-{{- if .Values.autoscaling.enabled }}
+{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}}
+{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}}
+{{- if and .Values.autoscaling.enabled (not $containerResourceAvailable) }}
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:

diff --git a/helm/charts/kratos/values.yaml b/helm/charts/kratos/values.yaml
@@ -531,6 +531,9 @@ autoscaling:
   # -- Set custom behavior
   # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior
   behavior: {}
+  # -- Add extraContainer container resource metrics
+  # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics
+  extraMetrics: []
 
 ## -- Values for initialization job
 job:

diff --git a/helm/charts/oathkeeper/templates/hpa-controller.yaml b/helm/charts/oathkeeper/templates/hpa-controller.yaml
@@ -1,4 +1,6 @@
-{{- if .Values.deployment.autoscaling.enabled }}
+{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}}
+{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}}
+{{- if and .Values.deployment.autoscaling.enabled (not $containerResourceAvailable) }}
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:

diff --git a/helm/charts/oathkeeper/templates/hpa.yaml b/helm/charts/oathkeeper/templates/hpa.yaml
@@ -0,0 +1,43 @@
+{{- $kubeVersion := .Capabilities.KubeVersion.GitVersion -}}
+{{- $containerResourceAvailable := and (not (empty $kubeVersion)) (semverCompare ">=1.3.0" $kubeVersion) -}}
+{{- if and .Values.deployment.autoscaling.enabled $containerResourceAvailable }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "oathkeeper.fullname" . }}
+  {{- if .Release.Namespace }}
+  namespace: {{ .Release.Namespace }}
+  {{- end }}
+  labels:
+    {{- include "oathkeeper.labels" . | nindent 4 }}
+spec:
+  {{- with .Values.deployment.autoscaling.behavior }}
+  behavior: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "oathkeeper.fullname" . }}
+  minReplicas: {{ .Values.deployment.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.deployment.autoscaling.maxReplicas }}
+  metrics:
+  {{- with .Values.deployment.autoscaling.targetMemory }}
+  - type: ContainerResource
+    containerResource:
+      name: memory
+      container: {{ $.Chart.Name }}
+      target:
+        {{- toYaml . | nindent 8 }}
+  {{- end }}
+  {{- with .Values.deployment.autoscaling.targetCPU}}
+  - type: ContainerResource
+    containerResource:
+      name: cpu
+      container: {{ $.Chart.Name }}
+      target:
+        {{- toYaml . | nindent 8 }}
+  {{- end }}
+  {{- if not (empty $.Values.deployment.autoscaling.extraMetrics) }}
+  {{- toYaml .Values.deployment.autoscaling.extraMetrics | nindent 2 }}
+  {{- end }}
+{{- end }}
diff --git a/helm/charts/oathkeeper/values.yaml b/helm/charts/oathkeeper/values.yaml
@@ -374,6 +374,9 @@ deployment:
     # -- Set custom behavior
     # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior
     behavior: {}
+    # -- Add extraContainer container resource metrics
+    # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#container-resource-metrics
+    extraMetrics: []
 
 # -- Configure node affinity
 affinity: {}