k8s runner: allow to override app selector label used in the service

Needed so we can assign workloads from separate deployments to the same service, therefore the same NEG.
grpc · Jul 26, 2024 · c92a5cd · c92a5cd
1 parent 101e6a3
commit c92a5cd
Show file tree

Hide file tree

Showing 12 changed files with 51 additions and 22 deletions.
diff --git a/framework/test_app/runners/k8s/gamma_server_runner.py b/framework/test_app/runners/k8s/gamma_server_runner.py
@@ -63,6 +63,7 @@ def __init__(
         deployment_name: str,
         image_name: str,
         td_bootstrap_image: str,
+        app_label: str = "",
         network: str = "default",
         xds_server_uri: Optional[str] = None,
         gcp_api_manager: gcp.api.GcpApiManager,
@@ -88,6 +89,7 @@ def __init__(
             deployment_name=deployment_name,
             image_name=image_name,
             td_bootstrap_image=td_bootstrap_image,
+            app_label=app_label,
             network=network,
             xds_server_uri=xds_server_uri,
             gcp_api_manager=gcp_api_manager,
@@ -154,7 +156,7 @@ def run(  # pylint: disable=arguments-differ
                 self.service_template,
                 service_name=self.service_name,
                 namespace_name=self.k8s_namespace.name,
-                deployment_name=self.deployment_name,
+                app_label=self.app_label,
                 neg_name=self.gcp_neg_name,
                 test_port=test_port,
             )
@@ -199,6 +201,7 @@ def run(  # pylint: disable=arguments-differ
             deployment_name=self.deployment_name,
             image_name=self.image_name,
             namespace_name=self.k8s_namespace.name,
+            app_label=self.app_label,
             service_account_name=self.service_account_name,
             td_bootstrap_image=self.td_bootstrap_image,
             xds_server_uri=self.xds_server_uri,

diff --git a/framework/test_app/runners/k8s/k8s_base_runner.py b/framework/test_app/runners/k8s/k8s_base_runner.py
@@ -84,6 +84,7 @@ class KubernetesBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta):
     # Required fields.
     k8s_namespace: k8s.KubernetesNamespace
     deployment_name: str
+    app_label: str
     image_name: str
     gcp_project: str
     gcp_service_account: str
@@ -141,6 +142,7 @@ def __init__(
         gcp_project: str,
         gcp_service_account: str,
         gcp_ui_url: str,
+        app_label: str = "",
         namespace_template: Optional[str] = "namespace.yaml",
         reuse_namespace: bool = False,
         address_type: Optional[str] = None,
@@ -151,6 +153,7 @@ def __init__(
         self.deployment_name = deployment_name
         self.image_name = image_name
         self.gcp_project = gcp_project
+        self.app_label = app_label or deployment_name
         # Maps GCP service account to Kubernetes service account
         self.gcp_service_account = gcp_service_account
         self.gcp_ui_url = gcp_ui_url

diff --git a/framework/test_app/runners/k8s/k8s_xds_client_runner.py b/framework/test_app/runners/k8s/k8s_xds_client_runner.py
@@ -64,6 +64,7 @@ def __init__(  # pylint: disable=too-many-locals
         deployment_name: str,
         image_name: str,
         td_bootstrap_image: str,
+        app_label: str = "",
         network="default",
         xds_server_uri: Optional[str] = None,
         gcp_api_manager: gcp.api.GcpApiManager,
@@ -84,6 +85,7 @@ def __init__(  # pylint: disable=too-many-locals
             deployment_name=deployment_name,
             image_name=image_name,
             gcp_project=gcp_project,
+            app_label=app_label,
             gcp_service_account=gcp_service_account,
             gcp_ui_url=gcp_api_manager.gcp_ui_url,
             namespace_template=namespace_template,
@@ -173,6 +175,7 @@ def run(  # pylint: disable=arguments-differ
             deployment_name=self.deployment_name,
             image_name=self.image_name,
             namespace_name=self.k8s_namespace.name,
+            app_label=self.app_label,
             service_account_name=self.service_account_name,
             td_bootstrap_image=self.td_bootstrap_image,
             xds_server_uri=self.xds_server_uri,

diff --git a/framework/test_app/runners/k8s/k8s_xds_server_runner.py b/framework/test_app/runners/k8s/k8s_xds_server_runner.py
@@ -91,6 +91,7 @@ def __init__(  # pylint: disable=too-many-locals
         deployment_name: str,
         image_name: str,
         td_bootstrap_image: str,
+        app_label: str = "",
         network: str = "default",
         xds_server_uri: Optional[str] = None,
         gcp_api_manager: gcp.api.GcpApiManager,
@@ -115,6 +116,7 @@ def __init__(  # pylint: disable=too-many-locals
             deployment_name=deployment_name,
             image_name=image_name,
             gcp_project=gcp_project,
+            app_label=app_label,
             gcp_service_account=gcp_service_account,
             gcp_ui_url=gcp_api_manager.gcp_ui_url,
             namespace_template=namespace_template,
@@ -223,7 +225,7 @@ def run(  # pylint: disable=arguments-differ,too-many-branches
                 self.service_template,
                 service_name=self.service_name,
                 namespace_name=self.k8s_namespace.name,
-                deployment_name=self.deployment_name,
+                app_label=self.app_label,
                 neg_name=self.gcp_neg_name,
                 test_port=test_port,
                 enable_dualstack=self.enable_dualstack,
@@ -253,6 +255,7 @@ def run(  # pylint: disable=arguments-differ,too-many-branches
             deployment_name=self.deployment_name,
             image_name=self.image_name,
             namespace_name=self.k8s_namespace.name,
+            app_label=self.app_label,
             service_account_name=self.service_account_name,
             td_bootstrap_image=self.td_bootstrap_image,
             xds_server_uri=self.xds_server_uri,

diff --git a/framework/test_app/server_app.py b/framework/test_app/server_app.py
@@ -63,10 +63,8 @@ def __init__(
         self.xds_host, self.xds_port = xds_host, xds_port
         self.monitoring_port = monitoring_port
 
-    def __str__(self):
-        return (
-            "XdsTestServer(ip=" + self.ip + " ,hostname=" + self.hostname + ")"
-        )
+    def __repr__(self):
+        return f"XdsTestServer(ip={self.ip}, hostname={self.hostname})"
 
     @property
     @functools.lru_cache(None)

diff --git a/kubernetes-manifests/client-secure.deployment.yaml b/kubernetes-manifests/client-secure.deployment.yaml
@@ -5,19 +5,19 @@ metadata:
   name: ${deployment_name}
   namespace: ${namespace_name}
   labels:
-    app: ${deployment_name}
+    app: ${app_label}
     deployment_id: ${deployment_id}
     owner: xds-k8s-interop-test
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: ${deployment_name}
+      app: ${app_label}
       deployment_id: ${deployment_id}
   template:
     metadata:
       labels:
-        app: ${deployment_name}
+        app: ${app_label}
         deployment_id: ${deployment_id}
         owner: xds-k8s-interop-test
       annotations:

diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml
@@ -5,19 +5,19 @@ metadata:
   name: ${deployment_name}
   namespace: ${namespace_name}
   labels:
-    app: ${deployment_name}
+    app: ${app_label}
     deployment_id: ${deployment_id}
     owner: xds-k8s-interop-test
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: ${deployment_name}
+      app: ${app_label}
       deployment_id: ${deployment_id}
   template:
     metadata:
       labels:
-        app: ${deployment_name}
+        app: ${app_label}
         deployment_id: ${deployment_id}
         owner: xds-k8s-interop-test
     spec:

diff --git a/kubernetes-manifests/gamma/service.yaml b/kubernetes-manifests/gamma/service.yaml
@@ -9,7 +9,7 @@ metadata:
 spec:
   type: ClusterIP
   selector:
-    app: ${deployment_name}
+    app: ${app_label}
   ports:
   - port: ${test_port}
     protocol: TCP

diff --git a/kubernetes-manifests/server-secure.deployment.yaml b/kubernetes-manifests/server-secure.deployment.yaml
@@ -5,21 +5,21 @@ metadata:
   name: ${deployment_name}
   namespace: ${namespace_name}
   labels:
-    app: ${deployment_name}
+    app: ${app_label}
     deployment_id: ${deployment_id}
     owner: xds-k8s-interop-test
 spec:
   replicas: ${replica_count}
   selector:
     matchLabels:
-      app: ${deployment_name}
+      app: ${app_label}
       deployment_id: ${deployment_id}
   template:
     metadata:
       annotations:
         security.cloud.google.com/use-workload-certificates: ""
       labels:
-        app: ${deployment_name}
+        app: ${app_label}
         deployment_id: ${deployment_id}
         owner: xds-k8s-interop-test
     spec:

diff --git a/kubernetes-manifests/server.deployment.yaml b/kubernetes-manifests/server.deployment.yaml
@@ -6,19 +6,19 @@ metadata:
   name: ${deployment_name}
   namespace: ${namespace_name}
   labels:
-    app: ${deployment_name}
+    app: ${app_label}
     deployment_id: ${deployment_id}
     owner: xds-k8s-interop-test
 spec:
   replicas: ${replica_count}
   selector:
     matchLabels:
-      app: ${deployment_name}
+      app: ${app_label}
       deployment_id: ${deployment_id}
   template:
     metadata:
       labels:
-        app: ${deployment_name}
+        app: ${app_label}
         deployment_id: ${deployment_id}
         owner: xds-k8s-interop-test
     spec:

diff --git a/kubernetes-manifests/server.service.yaml b/kubernetes-manifests/server.service.yaml
@@ -17,7 +17,7 @@ spec:
     - IPv4
   % endif
   selector:
-    app: ${deployment_name}
+    app: ${app_label}
   ports:
   - port: ${test_port}
     protocol: TCP

diff --git a/tests/dualstack_test.py b/tests/dualstack_test.py
@@ -32,6 +32,7 @@
 _Lang = skips.Lang
 
 _QPS = 100
+_SERVERS_APP_LABEL = "psm-interop-dualstack"
 
 
 class DualStackTest(xds_k8s_testcase.RegularXdsKubernetesTestCase):
@@ -63,18 +64,21 @@ def setUp(self):
         self.server_runner = _KubernetesServerRunner(
             self.server_runner.k8s_namespace,
             deployment_name=self.server_name,
+            app_label=_SERVERS_APP_LABEL,
             **runner_args,
         )
         self.v4_server_runner = _KubernetesServerRunner(
             self.server_runner.k8s_namespace,
             deployment_name=self.server_name + "-v4",
             service_name=self.server_runner.service_name,
+            app_label=_SERVERS_APP_LABEL,
             **runner_args,
         )
         self.v6_server_runner = _KubernetesServerRunner(
             self.server_runner.k8s_namespace,
             deployment_name=self.server_name + "-v6",
             service_name=self.server_runner.service_name,
+            app_label=_SERVERS_APP_LABEL,
             **runner_args,
         )
 
@@ -127,8 +131,23 @@ def test_dual_stack(self) -> None:
         logger.info(f"Test servers: {test_servers}")  # TODO: change to debug
 
         with self.subTest("04_add_server_backends_to_backend_services"):
-            self.setupServerBackends(server_runner=self.server_runner)
-            self.setupServerBackends(server_runner=self.v4_server_runner)
+            (
+                neg_name,
+                neg_zones,
+            ) = self.server_runner.k8s_namespace.parse_service_neg_status(
+                self.server_runner.service_name, self.server_port
+            )
+
+            # Add backends to the Backend Service
+            self.td.backend_service_add_neg_backends(
+                neg_name, neg_zones, max_rate_per_endpoint=5
+            )
+            self.td.wait_for_backends_healthy_status(
+                replica_count=len(test_servers)
+            )
+
+            # self.setupServerBackends(server_runner=self.server_runner)
+            # self.setupServerBackends(server_runner=self.v4_server_runner)
             # self.setupServerBackends(server_runner=self.v6_server_runner)
 
         test_client: _XdsTestClient