From 579f78256d55b80d2adfa55e83bce99edaf2ea6a Mon Sep 17 00:00:00 2001 From: eshitachandwani Date: Thu, 27 Feb 2025 01:38:29 +0530 Subject: [PATCH] running tests --- bin/lib/common.py | 8 +- bin/run_test_server_c6n.py | 16 +-- .../gcp/{c6n.py => cloud_run.py} | 8 +- framework/infrastructure/gcp/compute.py | 1 - .../infrastructure/gcp/network_services.py | 44 +++--- framework/infrastructure/traffic_director.py | 128 +----------------- .../traffic_director_cloudrun.py | 119 ++++++++++++++++ .../cloud_run_base_runner.py} | 33 ++--- .../cloud_run_xds_server_runner.py} | 17 ++- .../runners/k8s/k8s_xds_client_runner.py | 5 +- framework/xds_c6n_testcase.py | 0 framework/xds_flags.py | 2 +- framework/xds_k8s_testcase.py | 93 +++++++++++-- .../client.cloudrun.deployment.yaml | 74 ---------- kubernetes-manifests/client.deployment.yaml | 21 +++ ...r.py => baseline_test_cloud_run_server.py} | 2 +- tests/bootstrap_generator_test.py | 8 +- tests/url_map/affinity_test.py | 14 +- 18 files changed, 297 insertions(+), 296 deletions(-) rename framework/infrastructure/gcp/{c6n.py => cloud_run.py} (91%) create mode 100644 framework/infrastructure/traffic_director_cloudrun.py rename framework/test_app/runners/{c6n/c6n_base_runner.py => cloud_run/cloud_run_base_runner.py} (85%) rename framework/test_app/runners/{c6n/c6n_xds_server_runner.py => cloud_run/cloud_run_xds_server_runner.py} (88%) delete mode 100644 framework/xds_c6n_testcase.py delete mode 100644 kubernetes-manifests/client.cloudrun.deployment.yaml rename tests/{baseline_test_c6n_server.py => baseline_test_cloud_run_server.py} (95%) diff --git a/bin/lib/common.py b/bin/lib/common.py index 1f3fab49..5da21bd7 100644 --- a/bin/lib/common.py +++ b/bin/lib/common.py @@ -22,12 +22,12 @@ from framework import xds_flags from framework import xds_k8s_flags -from framework.infrastructure.gcp import c6n +from framework.infrastructure.gcp import cloud_run from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.test_app import client_app from framework.test_app import server_app -from framework.test_app.runners.c6n import c6n_xds_server_runner +from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner from framework.test_app.runners.k8s import gamma_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner @@ -47,7 +47,7 @@ KubernetesClientRunner = k8s_xds_client_runner.KubernetesClientRunner KubernetesServerRunner = k8s_xds_server_runner.KubernetesServerRunner GammaServerRunner = gamma_server_runner.GammaServerRunner -CloudRunServerRunner=c6n_xds_server_runner.CloudRunServerRunner +CloudRunServerRunner=cloud_run_xds_server_runner.CloudRunServerRunner _XdsTestServer = server_app.XdsTestServer _XdsTestClient = client_app.XdsTestClient @@ -64,7 +64,7 @@ def gcp_api_manager(): @functools.cache def c6n_api_manager(): - return c6n.CloudRunApiManager() + return cloud_run.CloudRunApiManager() def td_attrs(): return dict( diff --git a/bin/run_test_server_c6n.py b/bin/run_test_server_c6n.py index 7f842e80..c6b32253 100644 --- a/bin/run_test_server_c6n.py +++ b/bin/run_test_server_c6n.py @@ -17,23 +17,23 @@ Typical usage examples: # Help. - ./run.sh ./bin/run_test_server_c6n.py --help + ./run.sh ./bin/run_test_server_cloud_run.py --help # Run modes. - ./run.sh ./bin/run_test_server_c6n.py --mode=app_net - ./run.sh ./bin/run_test_server_c6n.py --mode=secure + ./run.sh ./bin/run_test_server_cloud_run.py --mode=app_net + ./run.sh ./bin/run_test_server_cloud_run.py --mode=secure # Gamma run mode: uses HTTPRoute by default. - ./run.sh ./bin/run_test_server_c6n.py --mode=gamma + ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma # Gamma run mode: use GRPCRoute. - ./run.sh ./bin/run_test_server_c6n.py --mode=gamma --gamma_route_kind=grpc + ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma --gamma_route_kind=grpc # Running multipler server replicas. - ./run.sh ./bin/run_test_server_c6n.py --server_replica_count=3 + ./run.sh ./bin/run_test_server_cloud_run.py --server_replica_count=3 # Cleanup: make sure to set the same mode used to create. - ./run.sh ./bin/run_test_server_c6n.py --mode=gamma --cmd=cleanup + ./run.sh ./bin/run_test_server_cloud_run.py --mode=gamma --cmd=cleanup """ import logging @@ -58,7 +58,7 @@ def main(argv): xds_flags.set_socket_default_timeout_from_flag() run_kwargs = dict() - server_runner = common.make_c6n_server_runner() + server_runner = common.make_cloud_run_server_runner() server_runner.run(**run_kwargs) diff --git a/framework/infrastructure/gcp/c6n.py b/framework/infrastructure/gcp/cloud_run.py similarity index 91% rename from framework/infrastructure/gcp/c6n.py rename to framework/infrastructure/gcp/cloud_run.py index 897b46db..204eeaa2 100644 --- a/framework/infrastructure/gcp/c6n.py +++ b/framework/infrastructure/gcp/cloud_run.py @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -#emchandwani : check import abc import logging @@ -58,8 +56,7 @@ def deploy_service(self, service_name: str, image_name: str, *,test_port: int = image=image_name, ports=[ run_v2.ContainerPort( - name="http1", - # emchandwani : change to self.server_xds_port + name="h2c", container_port=test_port, ), ], @@ -74,8 +71,7 @@ def deploy_service(self, service_name: str, image_name: str, *,test_port: int = try: operation = self._client.create_service(request=request) - # operation=GcpProjectApiResource.wait_for_operation(operation_request=request,test_success_fn=logger.info("done")) - self._service = operation.result(timeout=800) + self._service = operation.result(timeout=600) logger.info("Deployed service: %s", self._service.uri) return self._service.uri except Exception as e: diff --git a/framework/infrastructure/gcp/compute.py b/framework/infrastructure/gcp/compute.py index 34182a7c..92429c79 100644 --- a/framework/infrastructure/gcp/compute.py +++ b/framework/infrastructure/gcp/compute.py @@ -581,7 +581,6 @@ def create_serverless_neg( name: The name of the NEG. region: The region in which to create the NEG. service_name: The name of the Cloud Run service. Format: "namespaces/{namespace}/services/{service}" - network: The network of the NEG. Format: "projects/{project}/global/networks/{network}" Returns: The NEG selfLink URL diff --git a/framework/infrastructure/gcp/network_services.py b/framework/infrastructure/gcp/network_services.py index 09132536..d13e60fa 100644 --- a/framework/infrastructure/gcp/network_services.py +++ b/framework/infrastructure/gcp/network_services.py @@ -112,19 +112,14 @@ class RouteMatch: @classmethod def from_response(cls, d: Dict[str, Any]) -> "GrpcRoute.RouteMatch": return cls( - method=( - GrpcRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None - ), - headers=( - tuple( - GrpcRoute.HeaderMatch.from_response(h) - for h in d["headers"] - ) - if "headers" in d - else () - ), + method=GrpcRoute.MethodMatch.from_response(d["method"]) + if "method" in d + else None, + headers=tuple( + GrpcRoute.HeaderMatch.from_response(h) for h in d["headers"] + ) + if "headers" in d + else (), ) @dataclasses.dataclass(frozen=True) @@ -229,19 +224,14 @@ class RouteMatch: @classmethod def from_response(cls, d: Dict[str, Any]) -> "HttpRoute.RouteMatch": return cls( - method=( - HttpRoute.MethodMatch.from_response(d["method"]) - if "method" in d - else None - ), - headers=( - tuple( - HttpRoute.HeaderMatch.from_response(h) - for h in d["headers"] - ) - if "headers" in d - else () - ), + method=HttpRoute.MethodMatch.from_response(d["method"]) + if "method" in d + else None, + headers=tuple( + HttpRoute.HeaderMatch.from_response(h) for h in d["headers"] + ) + if "headers" in d + else (), ) @dataclasses.dataclass(frozen=True) @@ -484,4 +474,4 @@ def delete_http_route(self, name: str) -> bool: return self._delete_resource( collection=self._api_locations.httpRoutes(), full_name=self.resource_full_name(name, self.HTTP_ROUTES), - ) + ) \ No newline at end of file diff --git a/framework/infrastructure/traffic_director.py b/framework/infrastructure/traffic_director.py index 02d58d14..5d6b2f2a 100644 --- a/framework/infrastructure/traffic_director.py +++ b/framework/infrastructure/traffic_director.py @@ -1286,130 +1286,4 @@ def _get_certificate_provider(cls): "certificateProviderInstance": { "pluginInstance": cls.CERTIFICATE_PROVIDER_INSTANCE, }, - } - - -class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): - MESH_NAME = "grpc-mesh" - SERVER_TLS_POLICY_NAME = "server-tls-policy" - CLIENT_TLS_POLICY_NAME = "client-tls-policy" - AUTHZ_POLICY_NAME = "authz-policy" - ENDPOINT_POLICY = "endpoint-policy" - CERTIFICATE_PROVIDER_INSTANCE = "google_cloud_private_spiffe" - - netsec: _NetworkSecurityV1Beta1 - netsvc: _NetworkServicesV1Beta1 - - def __init__( - self, - gcp_api_manager: gcp.api.GcpApiManager, - project: str, - *, - resource_prefix: str, - resource_suffix: Optional[str] = None, - network: str = "default", - compute_api_version: str = "v1", - enable_dualstack: bool = False, - ): - super().__init__( - gcp_api_manager, - project, - resource_prefix=resource_prefix, - resource_suffix=resource_suffix, - network=network, - compute_api_version=compute_api_version, - enable_dualstack=enable_dualstack, - ) - - # API - # netsvc: gcp.network_services.NetworkServicesV1 - # self.netsvc = _NetworkServicesV1Beta1(gcp_api_manager, project) - # self.netsvc = gcp.network_services.NetworkServicesV1( - # gcp_api_manager, project - # ) - - # Managed resources - # TODO(gnossen) PTAL at the pylint error - self.grpc_route: Optional[GrpcRoute] = None - self.http_route: Optional[HttpRoute] = None - self.mesh: Optional[Mesh] = None - - # Managed resources - self.server_tls_policy: Optional[ServerTlsPolicy] = None - self.client_tls_policy: Optional[ClientTlsPolicy] = None - self.authz_policy: Optional[AuthorizationPolicy] = None - self.endpoint_policy: Optional[EndpointPolicy] = None - - def backend_service_add_backends( - self, - backends: Sequence[str], - region: Optional[str] = None, - balancing_mode: str = "CONNECTION", - max_rate_per_endpoint: Optional[int] = None, - capacity_scaler: float = 1.0, - *, - circuit_breakers: Optional[dict[str, int]] = None, - ): - - new_backends = [] - for backend in backends: - new_backend = { - "group": backend, - "balancingMode": balancing_mode, - "maxRatePerEndpoint": max_rate_per_endpoint, - "capacityScaler": capacity_scaler, - } - - if circuit_breakers is not None: - new_backend["circuitBreakers"] = circuit_breakers - - new_backends.append(new_backend) - - # self.backends.update(new_backends) - backend_service = self.backend_service - - logging.info( - "Adding backends to Backend Service %s: %r", - backend_service.name, - new_backends, - ) - - self.compute.backend_service_patch_backends( - backend_service, - backends, - is_cloudrun=True, - ) - - # def create_grpc_route(self, src_host: str, src_port: int): - # self.create_grpc_route(src_host, src_port) - # hostname = f"{src_host}:{src_port}" - # route_name = f"{self.resource_prefix}-grpc-route-{self.resource_suffix}" - # logger.info("Creating gRPC route: %s", route_name) - # service_name=self.netsvc.resource_full_name( - # self.backend_service.name, "backendServices" - # ) - # try: - # # backend_service_url = self.backend_service.url.split("v1/")[-1] - # # URL of the backend service - # route_body = { - # "name": route_name, - # "hostnames": [hostname], - # "rules": [ - # { - # "action": { - # "destinations": [ - # {"serviceName": service_name} - # ] - # }, - # } - # ], - # } - # if hasattr(self, "mesh") and self.mesh: - # route_body["meshes"] = self.mesh.url - # resource = self.netsvc.create_grpc_route(route_name, route_body) - # self.grpc_route = self.netsvc.get_grpc_route(route_name) - # logger.info("gRPC Route created successfully: %s", self.grpc_route) - - # except Exception as e: # Catching generic exceptions for now - # logger.exception("Error creating gRPC route: %s", e) - # raise + } \ No newline at end of file diff --git a/framework/infrastructure/traffic_director_cloudrun.py b/framework/infrastructure/traffic_director_cloudrun.py new file mode 100644 index 00000000..20ff2fa1 --- /dev/null +++ b/framework/infrastructure/traffic_director_cloudrun.py @@ -0,0 +1,119 @@ +# Copyright 2025 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +from typing import Optional, Sequence +from framework.infrastructure.gcp.network_security import NetworkSecurityV1Beta1 +from framework.infrastructure.gcp.network_services import NetworkServicesV1Beta1 +from framework.infrastructure.traffic_director import TrafficDirectorAppNetManager +from framework.infrastructure import gcp + +# _NetworkSecurityV1Beta1 = gcp.network_security.NetworkSecurityV1Beta1 +ServerTlsPolicy = gcp.network_security.ServerTlsPolicy +ClientTlsPolicy = gcp.network_security.ClientTlsPolicy +AuthorizationPolicy = gcp.network_security.AuthorizationPolicy + +# Network Services +# _NetworkServicesV1Beta1 = gcp.network_services.NetworkServicesV1Beta1 +EndpointPolicy = gcp.network_services.EndpointPolicy +GrpcRoute = gcp.network_services.GrpcRoute +HttpRoute = gcp.network_services.HttpRoute +Mesh = gcp.network_services.Mesh + +class TrafficDirectorCloudRunManager(TrafficDirectorAppNetManager): + MESH_NAME = "grpc-mesh" + SERVER_TLS_POLICY_NAME = "server-tls-policy" + CLIENT_TLS_POLICY_NAME = "client-tls-policy" + AUTHZ_POLICY_NAME = "authz-policy" + ENDPOINT_POLICY = "endpoint-policy" + CERTIFICATE_PROVIDER_INSTANCE = "google_cloud_private_spiffe" + + # netsec: _NetworkSecurityV1Beta1 + # netsvc: _NetworkServicesV1Beta1 + + def __init__( + self, + gcp_api_manager: gcp.api.GcpApiManager, + project: str, + *, + resource_prefix: str, + resource_suffix: Optional[str] = None, + network: str = "default", + compute_api_version: str = "v1", + enable_dualstack: bool = False, + ): + super().__init__( + gcp_api_manager, + project, + resource_prefix=resource_prefix, + resource_suffix=resource_suffix, + network=network, + compute_api_version=compute_api_version, + enable_dualstack=enable_dualstack, + ) + + # API + # self.netsvc = _NetworkServicesV1Beta1(gcp_api_manager, project) + # self.netsvc = _NetworkSecurityV1Beta1(gcp_api_manager, project) + + # Managed resources + # TODO(gnossen) PTAL at the pylint error + self.grpc_route: Optional[GrpcRoute] = None + self.http_route: Optional[HttpRoute] = None + self.mesh: Optional[Mesh] = None + + # Managed resources + self.server_tls_policy: Optional[ServerTlsPolicy] = None + self.client_tls_policy: Optional[ClientTlsPolicy] = None + self.authz_policy: Optional[AuthorizationPolicy] = None + self.endpoint_policy: Optional[EndpointPolicy] = None + + def backend_service_add_backends( + self, + backends: Sequence[str], + region: Optional[str] = None, + balancing_mode: str = "CONNECTION", + max_rate_per_endpoint: Optional[int] = None, + capacity_scaler: float = 1.0, + *, + circuit_breakers: Optional[dict[str, int]] = None, + ): + + new_backends = [] + for backend in backends: + new_backend = { + "group": backend, + "balancingMode": balancing_mode, + "maxRatePerEndpoint": max_rate_per_endpoint, + "capacityScaler": capacity_scaler, + } + + if circuit_breakers is not None: + new_backend["circuitBreakers"] = circuit_breakers + + new_backends.append(new_backend) + + # self.backends.update(new_backends) + backend_service = self.backend_service + + logging.info( + "Adding backends to Backend Service %s: %r", + backend_service.name, + new_backends, + ) + + self.compute.backend_service_patch_backends( + backend_service, + backends, + is_cloudrun=True, + ) \ No newline at end of file diff --git a/framework/test_app/runners/c6n/c6n_base_runner.py b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py similarity index 85% rename from framework/test_app/runners/c6n/c6n_base_runner.py rename to framework/test_app/runners/cloud_run/cloud_run_base_runner.py index efe37a3f..ab229fb5 100644 --- a/framework/test_app/runners/c6n/c6n_base_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_base_runner.py @@ -22,7 +22,7 @@ from typing import Optional import framework -from framework.infrastructure.gcp import c6n +from framework.infrastructure.gcp import cloud_run from framework.test_app.runners import base_runner logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ class CloudRunBaseRunner(base_runner.BaseRunner, metaclass=ABCMeta): image_name: str network: Optional[str] = None tag: str = "latest" - region: Optional[str] = None + region: str = "us-central1" current_revision: Optional[str] = None # gcp_project: Optional[str] = None gcp_ui_url: Optional[str] = None @@ -66,8 +66,8 @@ def __init__( project: str, service_name: str, image_name: str, + region: str, network: Optional[str] = None, - region: Optional[str] = None, ) -> None: super().__init__() @@ -77,7 +77,6 @@ def __init__( self.network = network self.region = region self.current_revision = None - # self.gcp_project = None self.gcp_ui_url = None # Persistent across many runs. @@ -93,7 +92,7 @@ def __init__( def _initalize_cloudrun_api_manager(self): """Initializes the CloudRunApiManager.""" - self.cloudrun_api_manager = c6n.CloudRunApiManager( + self.cloudrun_api_manager = cloud_run.CloudRunApiManager( project=self.project, region=self.region ) @@ -129,17 +128,13 @@ def _stop(self): ) self.run_history.append(run_history) - @classmethod - def _get_workload_identity_member_name( - cls, project, namespace_name, service_account_name - ): - """ - Returns workload identity member name used to authenticate Kubernetes - service accounts. - - https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity - """ - return ( - f"serviceAccount:{project}.svc.id.goog" - f"[{namespace_name}/{service_account_name}]" - ) + def stop(self): + """Deletes Cloud Run Service""" + logger.info("Deleting Cloud Run service: %s", self.service_name) + try: + self.cloudrun_api_manager.delete_service(self.service_name) + logger.info("Cloud Run service %s deleted", self.service_name) + except Exception as e: + logger.warning( + "Cloud Run service %s deletion failed: %s", self.service_name, e + ) \ No newline at end of file diff --git a/framework/test_app/runners/c6n/c6n_xds_server_runner.py b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py similarity index 88% rename from framework/test_app/runners/c6n/c6n_xds_server_runner.py rename to framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py index bd4003fc..12b29865 100644 --- a/framework/test_app/runners/c6n/c6n_xds_server_runner.py +++ b/framework/test_app/runners/cloud_run/cloud_run_xds_server_runner.py @@ -1,4 +1,4 @@ -# Copyright 2022 gRPC authors. +# Copyright 2025 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,8 +25,8 @@ from framework import xds_flags from framework import xds_k8s_flags -from framework.infrastructure.gcp import c6n -from framework.test_app.runners.c6n import c6n_base_runner +from framework.infrastructure.gcp import cloud_run +from framework.test_app.runners.cloud_run import cloud_run_base_runner from framework.test_app.server_app import XdsTestServer logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ def as_dict(self): } -class CloudRunServerRunner(c6n_base_runner.CloudRunBaseRunner): +class CloudRunServerRunner(cloud_run_base_runner.CloudRunBaseRunner): """Manages xDS Test Servers running on Cloud Run.""" def __init__( @@ -102,6 +102,9 @@ def get_service_url(self): @override def cleanup(self, *, force=False): - super().cleanup(force=force) - self._stop() - return + try: + if self.service: + self.stop() + self.service_name=None + finally: + self._stop() diff --git a/framework/test_app/runners/k8s/k8s_xds_client_runner.py b/framework/test_app/runners/k8s/k8s_xds_client_runner.py index b1f4e142..25430bac 100644 --- a/framework/test_app/runners/k8s/k8s_xds_client_runner.py +++ b/framework/test_app/runners/k8s/k8s_xds_client_runner.py @@ -131,12 +131,13 @@ def run( # pylint: disable=arguments-differ log_to_stdout: bool = False, request_payload_size: int = 0, response_payload_size: int = 0, + is_trusted_xds_server_experimental: bool=False ) -> client_app.XdsTestClient: logger.info( ( 'Deploying xDS test client "%s" to k8s namespace %s: ' "server_target=%s rpc=%s qps=%s metadata=%r secure_mode=%s " - "print_response=%s" + "print_response=%s is_trusted_xds_server_experimental=%s" ), self.deployment_name, self.k8s_namespace.name, @@ -146,6 +147,7 @@ def run( # pylint: disable=arguments-differ metadata, secure_mode, print_response, + is_trusted_xds_server_experimental ) super().run() @@ -191,6 +193,7 @@ def run( # pylint: disable=arguments-differ config_mesh=config_mesh, generate_mesh_id=generate_mesh_id, print_response=print_response, + is_trusted_xds_server_experimental=is_trusted_xds_server_experimental, **self.deployment_args.as_dict(), ) diff --git a/framework/xds_c6n_testcase.py b/framework/xds_c6n_testcase.py deleted file mode 100644 index e69de29b..00000000 diff --git a/framework/xds_flags.py b/framework/xds_flags.py index f601f6e8..b727e0de 100644 --- a/framework/xds_flags.py +++ b/framework/xds_flags.py @@ -201,7 +201,7 @@ REGION = flags.DEFINE_string( "region", - default="us-central1", + default=None, help="The region for deployment", ) diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py index d640a214..24560209 100644 --- a/framework/xds_k8s_testcase.py +++ b/framework/xds_k8s_testcase.py @@ -40,13 +40,14 @@ from framework.infrastructure import gcp from framework.infrastructure import k8s from framework.infrastructure import traffic_director +import framework.infrastructure.traffic_director_cloudrun as td_cloudrun from framework.infrastructure.gcp.compute import ComputeV1 from framework.rpc import grpc_channelz from framework.rpc import grpc_csds from framework.rpc import grpc_testing from framework.test_app import client_app from framework.test_app import server_app -from framework.test_app.runners.c6n import c6n_xds_server_runner +from framework.test_app.runners.cloud_run import cloud_run_xds_server_runner from framework.test_app.runners.k8s import k8s_xds_client_runner from framework.test_app.runners.k8s import k8s_xds_server_runner from framework.test_cases import base_testcase @@ -66,8 +67,8 @@ TrafficDirectorManager = traffic_director.TrafficDirectorManager TrafficDirectorAppNetManager = traffic_director.TrafficDirectorAppNetManager TrafficDirectorSecureManager = traffic_director.TrafficDirectorSecureManager -CloudRunServerRunner = c6n_xds_server_runner.CloudRunServerRunner -TrafficDirectorCloudRunManager = traffic_director.TrafficDirectorCloudRunManager +CloudRunServerRunner = cloud_run_xds_server_runner.CloudRunServerRunner +TrafficDirectorCloudRunManager = td_cloudrun.TrafficDirectorCloudRunManager XdsTestServer = server_app.XdsTestServer XdsTestClient = client_app.XdsTestClient ClientDeploymentArgs = k8s_xds_client_runner.ClientDeploymentArgs @@ -1404,7 +1405,6 @@ def debug_cert(cert): class CloudRunXdsKubernetesTestCase(RegularXdsKubernetesTestCase): server_runner: CloudRunServerRunner td: TrafficDirectorCloudRunManager - region: str @classmethod def setUpClass(cls): @@ -1479,11 +1479,16 @@ def startTestServers( return test_servers def startTestClient( - self, test_server: XdsTestServer, **kwargs - ) -> XdsTestClient: - # logger.info("emchandwani : test server : %v",test_server) - logger.info("emchandwani : xds URI : %s",test_server.xds_uri) - return self._start_test_client(test_server.xds_uri,secure_mode=True, **kwargs) + self, + test_server: XdsTestServer, + **kwargs, + ) -> XdsTestClient: + return self._start_test_client( + server_target=test_server.xds_uri, + secure_mode=True, + **kwargs, + ) + def backend_service_add_serverless_neg_backends(self): logger.info("Creating serverless NEG") @@ -1492,7 +1497,77 @@ def backend_service_add_serverless_neg_backends(self): self.region, self.server_namespace, ) + self.neg=neg return neg + + def assertXdsConfigExists(self, test_client: XdsTestClient): + config = test_client.csds.fetch_client_status(log_level=logging.INFO) + self.assertIsNotNone(config) + seen = set() + want = frozenset( + [ + "listener_config", + "cluster_config", + "route_config", + ] + ) + for xds_config in config.xds_config: + seen.add(xds_config.WhichOneof("per_xds_config")) + for generic_xds_config in config.generic_xds_configs: + if re.search(r"\.Listener$", generic_xds_config.type_url): + seen.add("listener_config") + elif re.search( + r"\.RouteConfiguration$", generic_xds_config.type_url + ): + seen.add("route_config") + elif re.search(r"\.Cluster$", generic_xds_config.type_url): + seen.add("cluster_config") + + logger.debug( + "Received xDS config dump: %s", + json_format.MessageToJson(config, indent=2), + ) + self.assertSameElements(want, seen) + + def cleanup(self): + self.td.cleanup(force=self.force_cleanup) + self.server_runner.cleanup(force=self.force_cleanup) + self.client_runner.cleanup( + force=self.force_cleanup, force_namespace=self.force_cleanup + ) + def tearDown(self): logger.info("----- TestMethod %s teardown -----", self.test_name) + logger.debug("Getting pods restart times") + client_restarts: int = 0 + try: + client_restarts = self.client_runner.get_pod_restarts( + self.client_runner.deployment + ) + except (retryers.RetryError, k8s.NotFound) as e: + logger.exception(e) + + retryer = retryers.constant_retryer( + wait_fixed=_timedelta(seconds=10), + attempts=3, + log_level=logging.INFO, + ) + try: + retryer(self.cleanup) + except retryers.RetryError: + logger.exception("Got error during teardown") + finally: + logger.info("----- Test client/server logs -----") + self.client_runner.logs_explorer_run_history_links() + + # Fail if any of the pods restarted. + self.assertEqual( + client_restarts, + 0, + msg=( + "Client container unexpectedly restarted" + f" {client_restarts} times during test. In most cases, this" + " is caused by the test client app crash." + ), + ) diff --git a/kubernetes-manifests/client.cloudrun.deployment.yaml b/kubernetes-manifests/client.cloudrun.deployment.yaml deleted file mode 100644 index 8c45c7ab..00000000 --- a/kubernetes-manifests/client.cloudrun.deployment.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: psm-grpc-client - labels: - app: psm-grpc-client -spec: - replicas: 1 - selector: - matchLabels: - app: psm-grpc-client - template: - metadata: - labels: - app: psm-grpc-client - spec: - % if service_account_name: - serviceAccountName: ${service_account_name} - % endif - containers: - - name: psm-grpc-client - image: ${image_name} - imagePullPolicy: Always - args: - - "--server=${server_target}" - env: - - name: GRPC_TRACE - value: "xds_client" - - name: GRPC_XDS_BOOTSTRAP - value: "/tmp/grpc-xds/td-grpc-bootstrap.json" - - name: GRPC_EXPERIMENTAL_XDS_AUTHORITY_REWRITE - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_SYSTEM_ROOT_CERTS - value: "true" - - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER - value: "true" - - - name: CSM_WORKLOAD_NAME - value: psm-grpc-client - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: NAMESPACE_NAME - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CONTAINER_NAME - value: psm-grpc-client - - name: OTEL_RESOURCE_ATTRIBUTES - value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME),k8s.container.name=$(CONTAINER_NAME) - volumeMounts: - - mountPath: /tmp/grpc-xds/ - name: grpc-td-conf - readOnly: true - initContainers: - - name: grpc-td-init - image: gcr.io/trafficdirector-prod/td-grpc-bootstrap:0.18.0 - imagePullPolicy: Always - args: - - "--output=/tmp/bootstrap/td-grpc-bootstrap.json" - - "--vpc-network-name=default" - - "--xds-server-uri=trafficdirector.googleapis.com:443" - - "--config-mesh=${config_mesh}" - - "--is-trusted-xds-server-experimental=true" - volumeMounts: - - mountPath: /tmp/bootstrap/ - name: grpc-td-conf - volumes: - - name: grpc-td-conf - emptyDir: - medium: Memory - -... diff --git a/kubernetes-manifests/client.deployment.yaml b/kubernetes-manifests/client.deployment.yaml index 0566575c..b5cfa2b9 100644 --- a/kubernetes-manifests/client.deployment.yaml +++ b/kubernetes-manifests/client.deployment.yaml @@ -41,6 +41,9 @@ spec: - "--qps=${qps}" - "--rpc=${rpc}" - "--metadata=${metadata}" + % if secure_mode: + - "--secure_mode=${secure_mode}" + % endif % if request_payload_size > 0: - "--request_payload_size=${request_payload_size}" % endif @@ -64,6 +67,22 @@ spec: value: "true" - name: GRPC_EXPERIMENTAL_XDS_GCP_AUTHENTICATION_FILTER value: "true" + - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH + value: "true" + - name: GRPC_XDS_EXPERIMENTAL_ENABLE_RETRY + value: "true" + - name: GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST + value: "true" + % if enable_dualstack: + - name: GRPC_EXPERIMENTAL_ENABLE_NEW_PICK_FIRST + value: "true" + - name: GRPC_EXPERIMENTAL_XDS_DUALSTACK_ENDPOINTS + value: "true" + % endif % if csm_workload_name: - name: CSM_WORKLOAD_NAME value: ${csm_workload_name} @@ -111,7 +130,9 @@ spec: % if generate_mesh_id: - "--generate-mesh-id-experimental" % endif + % if is_trusted_xds_server_experimental: - "--is-trusted-xds-server-experimental=true" + % endif resources: limits: cpu: 100m diff --git a/tests/baseline_test_c6n_server.py b/tests/baseline_test_cloud_run_server.py similarity index 95% rename from tests/baseline_test_c6n_server.py rename to tests/baseline_test_cloud_run_server.py index 9773893e..9295977f 100644 --- a/tests/baseline_test_c6n_server.py +++ b/tests/baseline_test_cloud_run_server.py @@ -49,7 +49,7 @@ def test_GKE_client_cloudrun_service(self): with self.subTest("7_start_test_client"): test_client: _XdsTestClient = self.startTestClient( - test_server, config_mesh=self.td.mesh.name + test_server, config_mesh=self.td.mesh.name,is_trusted_xds_server_experimental=True ) with self.subTest("8_test_client_xds_config_exists"): diff --git a/tests/bootstrap_generator_test.py b/tests/bootstrap_generator_test.py index 6e7bfa0a..c7b53ef3 100644 --- a/tests/bootstrap_generator_test.py +++ b/tests/bootstrap_generator_test.py @@ -38,9 +38,9 @@ # Constants GCR_PROD: Final[str] = "gcr.io/trafficdirector-prod/td-grpc-bootstrap" -GCR_TESTING: Final[str] = ( - "us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap" -) +GCR_TESTING: Final[ + str +] = "us-docker.pkg.dev/grpc-testing/trafficdirector/td-grpc-bootstrap" # Returns a list of bootstrap generator versions to be tested along with their @@ -230,4 +230,4 @@ def test_baseline_in_server_with_bootstrap_version(self, version, image): if __name__ == "__main__": - absltest.main() + absltest.main() \ No newline at end of file diff --git a/tests/url_map/affinity_test.py b/tests/url_map/affinity_test.py index e3e2102c..c180ccd1 100644 --- a/tests/url_map/affinity_test.py +++ b/tests/url_map/affinity_test.py @@ -101,9 +101,9 @@ def url_map_change( host_rule: HostRule, path_matcher: PathMatcher ) -> Tuple[HostRule, PathMatcher]: # Update default service to the affinity service. - path_matcher["defaultService"] = ( - GcpResourceManager().affinity_backend_service() - ) + path_matcher[ + "defaultService" + ] = GcpResourceManager().affinity_backend_service() return host_rule, path_matcher def xds_config_validate(self, xds_config: grpc_csds.DumpedXdsConfig): @@ -176,9 +176,9 @@ def url_map_change( host_rule: HostRule, path_matcher: PathMatcher ) -> Tuple[HostRule, PathMatcher]: # Update default service to the affinity service. - path_matcher["defaultService"] = ( - GcpResourceManager().affinity_backend_service() - ) + path_matcher[ + "defaultService" + ] = GcpResourceManager().affinity_backend_service() return host_rule, path_matcher def xds_config_validate(self, xds_config: grpc_csds.DumpedXdsConfig): @@ -273,4 +273,4 @@ def rpc_distribution_validate(self, test_client: XdsTestClient): # RPCs are sent to another backend if __name__ == "__main__": - absltest.main() + absltest.main() \ No newline at end of file