From 43c57e78177af2ec5b33cc83d3c4e4b0334f1735 Mon Sep 17 00:00:00 2001 From: "Jonathan Hess (he/him)" <103529393+hessjcg@users.noreply.github.com> Date: Tue, 21 Jan 2025 12:28:33 -0700 Subject: [PATCH] docs: Update kubernetes examples to use K8s native sidecar containers. (#2371) Running the proxy as a native sidecar container is now the preferred way to run the proxy. Using native sidecars makes the lifecycle much easier to manage: - Kubernetes will start the proxy container and wait until it is ready before it starts the application container. - Kubernetes will keep the proxy container running until the application container exits. - Kubernetes will stop the proxy container after the application container exits. - The exit status of the proxy will not impact the pod's overall exit status. This is especially important for Job and CronJob workloads. Native sidecar containers are supported by default in Kuberenetes 1.29. See K8s Sidecar Container: https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/ --- examples/k8s-health-check/README.md | 2 + .../proxy_with_http_health_check.yaml | 8 ++ examples/k8s-service/README.md | 2 + .../k8s-service/pgbouncer_deployment.yaml | 2 + examples/k8s-sidecar/README.md | 2 + .../k8s-sidecar/job_with_shutdown_hook.yaml | 7 ++ examples/k8s-sidecar/job_with_sidecar.yaml | 77 +++++++++++++++++++ examples/k8s-sidecar/proxy_with_sa_key.yaml | 2 + .../proxy_with_workload_identity.yaml | 2 + examples/multi-container/ruby/README.md | 2 + .../multi-container/ruby/multicontainer.yaml | 2 + 11 files changed, 108 insertions(+) create mode 100644 examples/k8s-sidecar/job_with_sidecar.yaml diff --git a/examples/k8s-health-check/README.md b/examples/k8s-health-check/README.md index 1c9c4f4b6..92875e353 100644 --- a/examples/k8s-health-check/README.md +++ b/examples/k8s-health-check/README.md @@ -172,7 +172,9 @@ for more than 1 minute. containers: - name: my-application image: gcr.io/my-container/my-application:1.1 + initContainers: - name: cloud-sql-proxy + restartPolicy: Always image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.14.3 args: # Set the --max-connections flag to 50 diff --git a/examples/k8s-health-check/proxy_with_http_health_check.yaml b/examples/k8s-health-check/proxy_with_http_health_check.yaml index adc5db29d..8b1b6c03d 100644 --- a/examples/k8s-health-check/proxy_with_http_health_check.yaml +++ b/examples/k8s-health-check/proxy_with_http_health_check.yaml @@ -46,7 +46,15 @@ spec: secretKeyRef: name: key: database + # The proxy should be run as a native sidecar container, available in + # Kubernetes 1.29 and higher. This will ensure that the proxy container + # is ready before the main application container is started, and + # that the proxy container's exit status will not impact the pod's exit + # status. See the Kubernetes documentation: + # https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/ + initContainers: - name: cloud-sql-proxy + restartPolicy: Always # It is recommended to use the latest version of the Cloud SQL Auth Proxy # Make sure to update on a regular schedule! image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.11.4 diff --git a/examples/k8s-service/README.md b/examples/k8s-service/README.md index dbbb778c7..fff627fe6 100644 --- a/examples/k8s-service/README.md +++ b/examples/k8s-service/README.md @@ -171,7 +171,9 @@ For the PgBouncer deployment, we add the proxy as a sidecar, starting it on port ``` yaml +initContainers: - name: cloud-sql-proxy + restartPolicy: Always image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.14.3 # make sure to use the latest version args: # Replace DB_PORT with the port the proxy should listen on diff --git a/examples/k8s-service/pgbouncer_deployment.yaml b/examples/k8s-service/pgbouncer_deployment.yaml index 516a2ba09..cf4eb9df5 100644 --- a/examples/k8s-service/pgbouncer_deployment.yaml +++ b/examples/k8s-service/pgbouncer_deployment.yaml @@ -81,7 +81,9 @@ spec: value: "/etc/server/key.pem" - name: CLIENT_TLS_CERT_FILE value: "/etc/server/cert.pem" + initContainers: - name: cloud-sql-proxy + restartPolicy: Always # It is recommended to use the latest version of the Cloud SQL Auth Proxy # Make sure to update on a regular schedule! image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.11.4 diff --git a/examples/k8s-sidecar/README.md b/examples/k8s-sidecar/README.md index 9d8d292fc..e04ca001f 100644 --- a/examples/k8s-sidecar/README.md +++ b/examples/k8s-sidecar/README.md @@ -180,7 +180,9 @@ as a separate service for several reasons: 1. Add the Cloud SQL Auth Proxy to the pod configuration under `containers`: > [proxy_with_workload-identity.yaml](proxy_with_workload_identity.yaml#L39-L69) ```yaml + initContainers: - name: cloud-sql-proxy + restartPolicy: Always # It is recommended to use the latest version of the Cloud SQL Auth Proxy # Make sure to update on a regular schedule! image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.14.3 # make sure to use the latest version diff --git a/examples/k8s-sidecar/job_with_shutdown_hook.yaml b/examples/k8s-sidecar/job_with_shutdown_hook.yaml index 5fbadeb70..6adb3d9de 100644 --- a/examples/k8s-sidecar/job_with_shutdown_hook.yaml +++ b/examples/k8s-sidecar/job_with_shutdown_hook.yaml @@ -41,6 +41,13 @@ spec: spec: containers: - name: my-application + # Note: This demonstrates a way to run the proxy in an older + # kubernetes cluster that does not support native sidecar containers. + # It is better to run the job as a native sidecar container. + # + # See the Kubernetes documentation: + # https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/ + # # Run your batch job command. # Then, send a HTTTP POST request to the proxy sidecar container's # /quitquitquit api. This will cause the proxy process to exit. diff --git a/examples/k8s-sidecar/job_with_sidecar.yaml b/examples/k8s-sidecar/job_with_sidecar.yaml new file mode 100644 index 000000000..a081a3d2d --- /dev/null +++ b/examples/k8s-sidecar/job_with_sidecar.yaml @@ -0,0 +1,77 @@ +# Copyright 2025 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### +# This demonstrates how to configure a batch job so that it shuts down +# the proxy containers when it has finished processing. +# +# This works in Kubernetes 1.29 and higher, demonstrating how to run the proxy +# using a native side-car container. +# +# See https://github.com/kubernetes/enhancements/issues/753 +# and https://github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/issues/381 + +apiVersion: batch/v1 +kind: Job +metadata: + name: job + labels: + app: busybox +spec: + template: + metadata: + creationTimestamp: null + labels: + app: busybox + spec: + containers: + - name: my-application + # Run your batch job command. + # Then, send a HTTTP POST request to the proxy sidecar container's + # /quitquitquit api. This will cause the proxy process to exit. + command: + - my_batch_job + - --host=127.0.0.1 + - --port= + - --username= + - --dbname= + image: my-application-image + imagePullPolicy: IfNotPresent + resources: {} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + restartPolicy: Never + terminationGracePeriodSeconds: 30 + initContainers: + - name: cloud-sql-proxy + restartPolicy: Always + # It is recommended to use the latest version of the Cloud SQL Auth Proxy + # Make sure to update on a regular schedule! + image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.14.3 + args: + # Enable the admin api server on port 9091 + - "--admin-port=9091" + + # Tell the proxy to exit gracefully if it receives a SIGTERM + - "--exit-zero-on-sigterm" + + # Replace DB_PORT with the port the proxy should listen on + - "--port=" + - "" + securityContext: + runAsNonRoot: true + resources: + requests: + memory: "2Gi" + cpu: "1" diff --git a/examples/k8s-sidecar/proxy_with_sa_key.yaml b/examples/k8s-sidecar/proxy_with_sa_key.yaml index 4a81d69e3..47351c090 100644 --- a/examples/k8s-sidecar/proxy_with_sa_key.yaml +++ b/examples/k8s-sidecar/proxy_with_sa_key.yaml @@ -44,7 +44,9 @@ spec: secretKeyRef: name: key: database + initContainers: - name: cloud-sql-proxy + restartPolicy: Always # It is recommended to use the latest version of the Cloud SQL Auth Proxy # Make sure to update on a regular schedule! image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.14.1 diff --git a/examples/k8s-sidecar/proxy_with_workload_identity.yaml b/examples/k8s-sidecar/proxy_with_workload_identity.yaml index 17b464cf4..7c68e7e68 100644 --- a/examples/k8s-sidecar/proxy_with_workload_identity.yaml +++ b/examples/k8s-sidecar/proxy_with_workload_identity.yaml @@ -50,7 +50,9 @@ spec: key: database # [END cloud_sql_proxy_k8s_secrets] # [START cloud_sql_proxy_k8s_container] + initContainers: - name: cloud-sql-proxy + restartPolicy: Always # It is recommended to use the latest version of the Cloud SQL Auth Proxy # Make sure to update on a regular schedule! image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.14.1 diff --git a/examples/multi-container/ruby/README.md b/examples/multi-container/ruby/README.md index 85720c5c2..460fbf066 100644 --- a/examples/multi-container/ruby/README.md +++ b/examples/multi-container/ruby/README.md @@ -98,7 +98,9 @@ spec: value: "127.0.0.1" - name: DB_PORT value: "5432" + initContainers: - name: cloud-sql-proxy + restartPolicy: Always image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:latest args: # Ensure the port number on the --port argument matches the value of diff --git a/examples/multi-container/ruby/multicontainer.yaml b/examples/multi-container/ruby/multicontainer.yaml index 99ac8cde0..68c2b005e 100644 --- a/examples/multi-container/ruby/multicontainer.yaml +++ b/examples/multi-container/ruby/multicontainer.yaml @@ -43,7 +43,9 @@ spec: value: "127.0.0.1" - name: DB_PORT value: "5432" + initContainers: - name: cloud-sql-proxy + restartPolicy: Always image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:latest args: # If connecting to a Cloud SQL instance within a VPC network, you can use the