From 9c45de830f0c6b4b46a8604178381a2dee864072 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Thu, 27 Oct 2022 10:20:53 +0200 Subject: [PATCH 01/26] Added vscode to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 3800b1cd..0534e444 100644 --- a/.gitignore +++ b/.gitignore @@ -106,6 +106,9 @@ ENV/ /.project /.pydevproject +# Visual Studio Code +.vscode + # Various openeo_driver/data tmp* From 009d68cf1d322aa126eb1e236497004a4075f514 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Thu, 27 Oct 2022 11:50:53 +0200 Subject: [PATCH 02/26] Issue #78 Implemented GET /service_types --- src/openeo_aggregator/backend.py | 51 ++++++++++++++++++++++- tests/test_backend.py | 70 ++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 70ddd709..8efc48ee 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -24,7 +24,7 @@ from openeo_aggregator.utils import MultiDictGetter, subdict, dict_merge, normalize_issuer_url from openeo_driver.ProcessGraphDeserializer import SimpleProcessing from openeo_driver.backend import OpenEoBackendImplementation, AbstractCollectionCatalog, LoadParameters, Processing, \ - OidcProvider, BatchJobs, BatchJobMetadata + OidcProvider, BatchJobs, BatchJobMetadata, SecondaryServices from openeo_driver.datacube import DriverDataCube from openeo_driver.errors import CollectionNotFoundException, OpenEOApiException, ProcessGraphMissingException, \ JobNotFoundException, JobNotFinishedException, ProcessGraphInvalidException, PermissionsInsufficientException, \ @@ -772,6 +772,47 @@ def get_log_entries(self, job_id: str, user_id: str, offset: Optional[str] = Non return con.job(backend_job_id).logs(offset=offset) +class AggregatorSecondaryServices(SecondaryServices): + """ + Aggregator implementation of the Secondary Services "microservice" + https://openeo.org/documentation/1.0/developers/api/reference.html#tag/Secondary-Services + """ + + def __init__( + self, + backends: MultiBackendConnection, + ): + super(AggregatorSecondaryServices, self).__init__() + self._backends = backends + + def service_types(self) -> dict: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-service-types""" + + service_types = {} + + def merge(formats: dict, to_add: dict): + # TODO: merge parameters in some way? + for name, data in to_add.items(): + if name.lower() not in {k.lower() for k in formats.keys()}: + formats[name] = data + + for con in self._backends: + try: + types_to_add = con.get("/service_types").json() + except Exception as e: + # TODO: fail instead of warn? + _log.warning(f"Failed to get service_types from {con.id}: {e!r}", exc_info=True) + continue + # TODO #1 smarter merging: parameter differences? + merge(service_types, types_to_add) + + return service_types + + # next one to implement + # def list_services(self, user_id: str) -> List[ServiceMetadata]: + # """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-services""" + # return [] + class AggregatorBackendImplementation(OpenEoBackendImplementation): # No basic auth: OIDC auth is required (to get EGI Check-in eduperson_entitlement data) enable_basic_auth = False @@ -797,10 +838,13 @@ def __init__(self, backends: MultiBackendConnection, config: AggregatorConfig): processing=processing, partitioned_job_tracker=partitioned_job_tracker ) + + secondary_services = AggregatorSecondaryServices(backends=backends) + super().__init__( catalog=catalog, processing=processing, - secondary_services=None, + secondary_services=secondary_services, batch_jobs=batch_jobs, user_defined_processes=None, ) @@ -953,3 +997,6 @@ def postprocess_capabilities(self, capabilities: dict) -> dict: # TODO: standardize this field? capabilities["_partitioned_job_tracking"] = bool(self.batch_jobs.partitioned_job_tracker) return capabilities + + def service_types(self) -> dict: + return self.secondary_services.service_types() diff --git a/tests/test_backend.py b/tests/test_backend.py index 301d1bd2..255b2db4 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -110,6 +110,76 @@ def test_file_formats_merging(self, multi_backend_connection, config, backend1, } } + def test_service_types_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): + single_service_type = { + "WMTS": { + "configuration": { + "colormap": { + "default": "YlGn", + "description": + "The colormap to apply to single band layers", + "type": "string" + }, + "version": { + "default": "1.0.0", + "description": "The WMTS version to use.", + "enum": ["1.0.0"], + "type": "string" + } + }, + "links": [], + "process_parameters": [], + "title": "Web Map Tile Service" + } + } + requests_mock.get(backend1 + "/service_types", json=single_service_type) + requests_mock.get(backend2 + "/service_types", json=single_service_type) + implementation = AggregatorBackendImplementation( + backends=multi_backend_connection, config=config + ) + service_types = implementation.service_types() + assert service_types == single_service_type + + def test_service_types_merging(self, multi_backend_connection, config, backend1, backend2, requests_mock): + service_1 = { + "WMTS": { + "configuration": { + "colormap": { + "default": "YlGn", + "description": + "The colormap to apply to single band layers", + "type": "string" + }, + "version": { + "default": "1.0.0", + "description": "The WMTS version to use.", + "enum": ["1.0.0"], + "type": "string" + } + }, + "links": [], + "process_parameters": [], + "title": "Web Map Tile Service" + } + } + service_2 = { + "WMS": { + "title": "OGC Web Map Service", + "configuration": {}, + "process_parameters": [], + "links": [] + } + } + requests_mock.get(backend1 + "/service_types", json=service_1) + requests_mock.get(backend2 + "/service_types", json=service_2) + implementation = AggregatorBackendImplementation( + backends=multi_backend_connection, config=config + ) + service_types = implementation.service_types() + expected = dict(service_1) + expected.update(service_2) + assert service_types == expected + class TestInternalCollectionMetadata: From 466616e7809f9748ded04601c1080092ad0a85ef Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Wed, 2 Nov 2022 10:24:35 +0100 Subject: [PATCH 03/26] Issue #78 Implementing GET /services --- src/openeo_aggregator/backend.py | 33 +- tests/test_backend.py | 719 +++++++++++++++++++++++++++++++ 2 files changed, 747 insertions(+), 5 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 8efc48ee..37ff8293 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -24,7 +24,7 @@ from openeo_aggregator.utils import MultiDictGetter, subdict, dict_merge, normalize_issuer_url from openeo_driver.ProcessGraphDeserializer import SimpleProcessing from openeo_driver.backend import OpenEoBackendImplementation, AbstractCollectionCatalog, LoadParameters, Processing, \ - OidcProvider, BatchJobs, BatchJobMetadata, SecondaryServices + OidcProvider, BatchJobs, BatchJobMetadata, SecondaryServices, ServiceMetadata from openeo_driver.datacube import DriverDataCube from openeo_driver.errors import CollectionNotFoundException, OpenEOApiException, ProcessGraphMissingException, \ JobNotFoundException, JobNotFinishedException, ProcessGraphInvalidException, PermissionsInsufficientException, \ @@ -808,10 +808,30 @@ def merge(formats: dict, to_add: dict): return service_types - # next one to implement - # def list_services(self, user_id: str) -> List[ServiceMetadata]: - # """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-services""" - # return [] + def list_services(self, user_id: str) -> List[ServiceMetadata]: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-services""" + + all_services = [] + def merge(services, to_add): + # For now ignore the links + services_to_add = to_add.get("services") + if services_to_add: + services_metadata = [ServiceMetadata.from_dict(s) for s in services_to_add] + services.extend(services_metadata) + + # Get stuff from backends + for con in self._backends: + services_json = None + try: + services_json = con.get("/services").json() + except Exception as e: + _log.warning("Failed to get services from {con.id}: {e!r}", exc_info=True) + continue + + if services_json: + merge(all_services, services_json) + + return all_services class AggregatorBackendImplementation(OpenEoBackendImplementation): # No basic auth: OIDC auth is required (to get EGI Check-in eduperson_entitlement data) @@ -1000,3 +1020,6 @@ def postprocess_capabilities(self, capabilities: dict) -> dict: def service_types(self) -> dict: return self.secondary_services.service_types() + + def list_services(self, user_id: str) -> List[ServiceMetadata]: + return self.secondary_services.list_services(user_id=user_id) diff --git a/tests/test_backend.py b/tests/test_backend.py index 255b2db4..f3e47269 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,9 +1,13 @@ +from sys import implementation +from datetime import datetime + import pytest from openeo_aggregator.backend import AggregatorCollectionCatalog, AggregatorProcessing, \ AggregatorBackendImplementation, _InternalCollectionMetadata, JobIdMapping from openeo_aggregator.caching import DictMemoizer from openeo_aggregator.testing import clock_mock +from openeo_driver.backend import ServiceMetadata from openeo_driver.errors import OpenEOApiException, CollectionNotFoundException, JobNotFoundException from openeo_driver.testing import DictSubSet from openeo_driver.users.oidc import OidcProvider @@ -180,6 +184,721 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, expected.update(service_2) assert service_types == expected + TEST_SERVICES = { + "services": [{ + "id": "wms-a3cca9", + "title": "NDVI based on Sentinel 2", + "description": "Deriving minimum NDVI measurements over pixel time series of Sentinel 2", + "url": "https://example.openeo.org/wms/wms-a3cca9", + "type": "wms", + "enabled": True, + "process": { + "id": "ndvi", + "summary": "string", + "description": "string", + "parameters": [{ + "schema": { + "parameters": [{ + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "name": "string", + "description": "string", + "optional": False, + "deprecated": False, + "experimental": False, + "default": None + }], + "returns": { + "description": "string", + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "name": "string", + "description": "string", + "optional": False, + "deprecated": False, + "experimental": False, + "default": None + }], + "returns": { + "description": "string", + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + } + }, + "categories": ["string"], + "deprecated": False, + "experimental": False, + "exceptions": { + "Error Code1": { + "description": "string", + "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", + "http": 400 + }, + "Error Code2": { + "description": "string", + "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", + "http": 400 + } + }, + "examples": [{ + "title": "string", + "description": "string", + "arguments": { + "property1": { + "from_parameter": None, + "from_node": None, + "process_graph": None + }, + "property2": { + "from_parameter": None, + "from_node": None, + "process_graph": None + } + }, + "returns": None + }], + "links": [{ + "rel": "related", + "href": "https://example.openeo.org", + "type": "text/html", + "title": "openEO" + }], + "process_graph": { + "dc": { + "process_id": "load_collection", + "arguments": { + "id": "Sentinel-2", + "spatial_extent": { + "west": 16.1, + "east": 16.6, + "north": 48.6, + "south": 47.2 + }, + "temporal_extent": ["2018-01-01", "2018-02-01"] + } + }, + "bands": { + "process_id": "filter_bands", + "description": + "Filter and order the bands. The order is important for the following reduce operation.", + "arguments": { + "data": { + "from_node": "dc" + }, + "bands": ["B08", "B04", "B02"] + } + }, + "evi": { + "process_id": "reduce", + "description": + "Compute the EVI. Formula: 2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)", + "arguments": { + "data": { + "from_node": "bands" + }, + "dimension": "bands", + "reducer": { + "process_graph": { + "nir": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 0 + } + }, + "red": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 1 + } + }, + "blue": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 2 + } + }, + "sub": { + "process_id": "subtract", + "arguments": { + "data": [{ + "from_node": "nir" + }, { + "from_node": "red" + }] + } + }, + "p1": { + "process_id": "product", + "arguments": { + "data": [6, { + "from_node": "red" + }] + } + }, + "p2": { + "process_id": "product", + "arguments": { + "data": + [-7.5, { + "from_node": "blue" + }] + } + }, + "sum": { + "process_id": "sum", + "arguments": { + "data": [ + 1, { + "from_node": "nir" + }, { + "from_node": "p1" + }, { + "from_node": "p2" + } + ] + } + }, + "div": { + "process_id": "divide", + "arguments": { + "data": [{ + "from_node": "sub" + }, { + "from_node": "sum" + }] + } + }, + "p3": { + "process_id": "product", + "arguments": { + "data": + [2.5, { + "from_node": "div" + }] + }, + "result": True + } + } + } + } + }, + "mintime": { + "process_id": "reduce", + "description": + "Compute a minimum time composite by reducing the temporal dimension", + "arguments": { + "data": { + "from_node": "evi" + }, + "dimension": "temporal", + "reducer": { + "process_graph": { + "min": { + "process_id": "min", + "arguments": { + "data": { + "from_parameter": "data" + } + }, + "result": True + } + } + } + } + }, + "save": { + "process_id": "save_result", + "arguments": { + "data": { + "from_node": "mintime" + }, + "format": "GTiff" + }, + "result": True + } + } + }, + "configuration": { + "version": "1.3.0" + }, + "attributes": { + "layers": ["ndvi", "evi"] + }, + "created": "2017-01-01T09:32:12Z", + "plan": "free", + "costs": 12.98, + "budget": 100, + "usage": { + "cpu": { + "value": 40668, + "unit": "cpu-seconds" + }, + "duration": { + "value": 2611, + "unit": "seconds" + }, + "memory": { + "value": 108138811, + "unit": "mb-seconds" + }, + "network": { + "value": 0, + "unit": "kb" + }, + "storage": { + "value": 55, + "unit": "mb" + } + } + }], + "links": [{ + "rel": "related", + "href": "https://example.openeo.org", + "type": "text/html", + "title": "openEO" + }] + } + + TEST_SERVICES2 = { + "services": [{ + "id": "wms-a3cca9", + "title": "TEST COPY -- NDVI based on Sentinel 2", + "description": "TEST COPY Deriving minimum NDVI measurements over pixel time series of Sentinel 2", + "url": "https://example.openeo.org/wms/wms-a3cca9", + "type": "wms", + "enabled": True, + "process": { + "id": "ndvi", + "summary": "string", + "description": "string", + "parameters": [{ + "schema": { + "parameters": [{ + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "name": "string", + "description": "string", + "optional": False, + "deprecated": False, + "experimental": False, + "default": None + }], + "returns": { + "description": "string", + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "name": "string", + "description": "string", + "optional": False, + "deprecated": False, + "experimental": False, + "default": None + }], + "returns": { + "description": "string", + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + } + }, + "categories": ["string"], + "deprecated": False, + "experimental": False, + "exceptions": { + "Error Code1": { + "description": "string", + "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", + "http": 400 + }, + "Error Code2": { + "description": "string", + "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", + "http": 400 + } + }, + "examples": [{ + "title": "string", + "description": "string", + "arguments": { + "property1": { + "from_parameter": None, + "from_node": None, + "process_graph": None + }, + "property2": { + "from_parameter": None, + "from_node": None, + "process_graph": None + } + }, + "returns": None + }], + "links": [{ + "rel": "related", + "href": "https://example.openeo.org", + "type": "text/html", + "title": "openEO" + }], + "process_graph": { + "dc": { + "process_id": "load_collection", + "arguments": { + "id": "Sentinel-2", + "spatial_extent": { + "west": 16.1, + "east": 16.6, + "north": 48.6, + "south": 47.2 + }, + "temporal_extent": ["2018-01-01", "2018-02-01"] + } + }, + "bands": { + "process_id": "filter_bands", + "description": + "Filter and order the bands. The order is important for the following reduce operation.", + "arguments": { + "data": { + "from_node": "dc" + }, + "bands": ["B08", "B04", "B02"] + } + }, + "evi": { + "process_id": "reduce", + "description": + "Compute the EVI. Formula: 2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)", + "arguments": { + "data": { + "from_node": "bands" + }, + "dimension": "bands", + "reducer": { + "process_graph": { + "nir": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 0 + } + }, + "red": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 1 + } + }, + "blue": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 2 + } + }, + "sub": { + "process_id": "subtract", + "arguments": { + "data": [{ + "from_node": "nir" + }, { + "from_node": "red" + }] + } + }, + "p1": { + "process_id": "product", + "arguments": { + "data": [6, { + "from_node": "red" + }] + } + }, + "p2": { + "process_id": "product", + "arguments": { + "data": + [-7.5, { + "from_node": "blue" + }] + } + }, + "sum": { + "process_id": "sum", + "arguments": { + "data": [ + 1, { + "from_node": "nir" + }, { + "from_node": "p1" + }, { + "from_node": "p2" + } + ] + } + }, + "div": { + "process_id": "divide", + "arguments": { + "data": [{ + "from_node": "sub" + }, { + "from_node": "sum" + }] + } + }, + "p3": { + "process_id": "product", + "arguments": { + "data": + [2.5, { + "from_node": "div" + }] + }, + "result": True + } + } + } + } + }, + "mintime": { + "process_id": "reduce", + "description": + "Compute a minimum time composite by reducing the temporal dimension", + "arguments": { + "data": { + "from_node": "evi" + }, + "dimension": "temporal", + "reducer": { + "process_graph": { + "min": { + "process_id": "min", + "arguments": { + "data": { + "from_parameter": "data" + } + }, + "result": True + } + } + } + } + }, + "save": { + "process_id": "save_result", + "arguments": { + "data": { + "from_node": "mintime" + }, + "format": "GTiff" + }, + "result": True + } + } + }, + "configuration": { + "version": "1.3.0" + }, + "attributes": { + "layers": ["ndvi", "evi"] + }, + "created": "2017-01-01T09:32:12Z", + "plan": "free", + "costs": 12.98, + "budget": 100, + "usage": { + "cpu": { + "value": 40668, + "unit": "cpu-seconds" + }, + "duration": { + "value": 2611, + "unit": "seconds" + }, + "memory": { + "value": 108138811, + "unit": "mb-seconds" + }, + "network": { + "value": 0, + "unit": "kb" + }, + "storage": { + "value": 55, + "unit": "mb" + } + } + }], + "links": [] + } + + def test_list_services_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): + services1 = self.TEST_SERVICES + services2 = {} + test_user_id = "fakeuser" + requests_mock.get(backend1 + "/services", json=services1) + requests_mock.get(backend2 + "/services", json=services2) + implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + actual_services = implementation.list_services(user_id=test_user_id) + + # Construct expected result. We have get just data from the service in services1 + # (there is only one) for conversion to a ServiceMetadata. + the_service = services1["services"][0] + expected_services = [ + ServiceMetadata.from_dict(the_service) + ] + assert actual_services == expected_services + + def test_list_services_merged(self, multi_backend_connection, config, backend1, backend2, requests_mock): + services1 = self.TEST_SERVICES + serv_metadata_wmts_foo = ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test service", + created=datetime(2020, 4, 9, 15, 5, 8) + ) + services2 = {"services": [serv_metadata_wmts_foo.prepare_for_json()], "links": []} + test_user_id = "fakeuser" + requests_mock.get(backend1 + "/services", json=services1) + requests_mock.get(backend2 + "/services", json=services2) + implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + actual_services = implementation.list_services(user_id=test_user_id) + + # Construct expected result. We have get just data from the service in + # services1 (there is only one) for conversion to a ServiceMetadata. + # For now we still ignore the key "links" in the outer dictionary. + service1 = services1["services"][0] + service1_md = ServiceMetadata.from_dict(service1) + service2 = services2["services"][0] + service2_md = ServiceMetadata.from_dict(service2) + expected_services = [service1_md, service2_md] + + assert actual_services == expected_services + class TestInternalCollectionMetadata: From 88b7e6c0336ffdf831753a49db48b0156df9c24a Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Wed, 2 Nov 2022 20:17:41 +0100 Subject: [PATCH 04/26] Issue #78 Implementing service_info - GET /services/{service_id} --- src/openeo_aggregator/backend.py | 25 +- tests/test_backend.py | 469 +++++++++---------------------- 2 files changed, 152 insertions(+), 342 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 37ff8293..b5b6e9de 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -28,7 +28,7 @@ from openeo_driver.datacube import DriverDataCube from openeo_driver.errors import CollectionNotFoundException, OpenEOApiException, ProcessGraphMissingException, \ JobNotFoundException, JobNotFinishedException, ProcessGraphInvalidException, PermissionsInsufficientException, \ - FeatureUnsupportedException + FeatureUnsupportedException, ServiceNotFoundException from openeo_driver.processes import ProcessRegistry from openeo_driver.users import User from openeo_driver.utils import EvalEnv @@ -791,11 +791,11 @@ def service_types(self) -> dict: service_types = {} def merge(formats: dict, to_add: dict): - # TODO: merge parameters in some way? for name, data in to_add.items(): if name.lower() not in {k.lower() for k in formats.keys()}: formats[name] = data + # Collect all service types from the backends. for con in self._backends: try: types_to_add = con.get("/service_types").json() @@ -819,7 +819,7 @@ def merge(services, to_add): services_metadata = [ServiceMetadata.from_dict(s) for s in services_to_add] services.extend(services_metadata) - # Get stuff from backends + # Collect all services from the backends. for con in self._backends: services_json = None try: @@ -833,6 +833,22 @@ def merge(services, to_add): return all_services + def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-service""" + + # TODO: can there ever be a service with the same ID in multiple back-ends? (For the same user) + for con in self._backends: + try: + service_json = con.get(f"/services/{service_id}").json() + except Exception as e: + _log.debug("No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) + continue + else: + return ServiceMetadata.from_dict(service_json) + + raise ServiceNotFoundException(service_id) + + class AggregatorBackendImplementation(OpenEoBackendImplementation): # No basic auth: OIDC auth is required (to get EGI Check-in eduperson_entitlement data) enable_basic_auth = False @@ -1023,3 +1039,6 @@ def service_types(self) -> dict: def list_services(self, user_id: str) -> List[ServiceMetadata]: return self.secondary_services.list_services(user_id=user_id) + + def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: + return self.secondary_services.service_info(user_id=user_id, service_id=service_id) diff --git a/tests/test_backend.py b/tests/test_backend.py index f3e47269..9f8d7ae0 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -8,7 +8,8 @@ from openeo_aggregator.caching import DictMemoizer from openeo_aggregator.testing import clock_mock from openeo_driver.backend import ServiceMetadata -from openeo_driver.errors import OpenEOApiException, CollectionNotFoundException, JobNotFoundException +from openeo_driver.errors import OpenEOApiException, CollectionNotFoundException, JobNotFoundException, \ + ServiceNotFoundException from openeo_driver.testing import DictSubSet from openeo_driver.users.oidc import OidcProvider from .conftest import DEFAULT_MEMOIZER_CONFIG @@ -519,344 +520,14 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, }] } - TEST_SERVICES2 = { - "services": [{ - "id": "wms-a3cca9", - "title": "TEST COPY -- NDVI based on Sentinel 2", - "description": "TEST COPY Deriving minimum NDVI measurements over pixel time series of Sentinel 2", - "url": "https://example.openeo.org/wms/wms-a3cca9", - "type": "wms", - "enabled": True, - "process": { - "id": "ndvi", - "summary": "string", - "description": "string", - "parameters": [{ - "schema": { - "parameters": [{ - "schema": { - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - }, - "name": "string", - "description": "string", - "optional": False, - "deprecated": False, - "experimental": False, - "default": None - }], - "returns": { - "description": "string", - "schema": { - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - }, - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - }, - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - }, - "name": "string", - "description": "string", - "optional": False, - "deprecated": False, - "experimental": False, - "default": None - }], - "returns": { - "description": "string", - "schema": { - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - } - }, - "categories": ["string"], - "deprecated": False, - "experimental": False, - "exceptions": { - "Error Code1": { - "description": "string", - "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", - "http": 400 - }, - "Error Code2": { - "description": "string", - "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", - "http": 400 - } - }, - "examples": [{ - "title": "string", - "description": "string", - "arguments": { - "property1": { - "from_parameter": None, - "from_node": None, - "process_graph": None - }, - "property2": { - "from_parameter": None, - "from_node": None, - "process_graph": None - } - }, - "returns": None - }], - "links": [{ - "rel": "related", - "href": "https://example.openeo.org", - "type": "text/html", - "title": "openEO" - }], - "process_graph": { - "dc": { - "process_id": "load_collection", - "arguments": { - "id": "Sentinel-2", - "spatial_extent": { - "west": 16.1, - "east": 16.6, - "north": 48.6, - "south": 47.2 - }, - "temporal_extent": ["2018-01-01", "2018-02-01"] - } - }, - "bands": { - "process_id": "filter_bands", - "description": - "Filter and order the bands. The order is important for the following reduce operation.", - "arguments": { - "data": { - "from_node": "dc" - }, - "bands": ["B08", "B04", "B02"] - } - }, - "evi": { - "process_id": "reduce", - "description": - "Compute the EVI. Formula: 2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)", - "arguments": { - "data": { - "from_node": "bands" - }, - "dimension": "bands", - "reducer": { - "process_graph": { - "nir": { - "process_id": "array_element", - "arguments": { - "data": { - "from_parameter": "data" - }, - "index": 0 - } - }, - "red": { - "process_id": "array_element", - "arguments": { - "data": { - "from_parameter": "data" - }, - "index": 1 - } - }, - "blue": { - "process_id": "array_element", - "arguments": { - "data": { - "from_parameter": "data" - }, - "index": 2 - } - }, - "sub": { - "process_id": "subtract", - "arguments": { - "data": [{ - "from_node": "nir" - }, { - "from_node": "red" - }] - } - }, - "p1": { - "process_id": "product", - "arguments": { - "data": [6, { - "from_node": "red" - }] - } - }, - "p2": { - "process_id": "product", - "arguments": { - "data": - [-7.5, { - "from_node": "blue" - }] - } - }, - "sum": { - "process_id": "sum", - "arguments": { - "data": [ - 1, { - "from_node": "nir" - }, { - "from_node": "p1" - }, { - "from_node": "p2" - } - ] - } - }, - "div": { - "process_id": "divide", - "arguments": { - "data": [{ - "from_node": "sub" - }, { - "from_node": "sum" - }] - } - }, - "p3": { - "process_id": "product", - "arguments": { - "data": - [2.5, { - "from_node": "div" - }] - }, - "result": True - } - } - } - } - }, - "mintime": { - "process_id": "reduce", - "description": - "Compute a minimum time composite by reducing the temporal dimension", - "arguments": { - "data": { - "from_node": "evi" - }, - "dimension": "temporal", - "reducer": { - "process_graph": { - "min": { - "process_id": "min", - "arguments": { - "data": { - "from_parameter": "data" - } - }, - "result": True - } - } - } - } - }, - "save": { - "process_id": "save_result", - "arguments": { - "data": { - "from_node": "mintime" - }, - "format": "GTiff" - }, - "result": True - } - } - }, - "configuration": { - "version": "1.3.0" - }, - "attributes": { - "layers": ["ndvi", "evi"] - }, - "created": "2017-01-01T09:32:12Z", - "plan": "free", - "costs": 12.98, - "budget": 100, - "usage": { - "cpu": { - "value": 40668, - "unit": "cpu-seconds" - }, - "duration": { - "value": 2611, - "unit": "seconds" - }, - "memory": { - "value": 108138811, - "unit": "mb-seconds" - }, - "network": { - "value": 0, - "unit": "kb" - }, - "storage": { - "value": 55, - "unit": "mb" - } - } - }], - "links": [] - } - def test_list_services_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): services1 = self.TEST_SERVICES services2 = {} - test_user_id = "fakeuser" requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + test_user_id = "fakeuser" actual_services = implementation.list_services(user_id=test_user_id) # Construct expected result. We have get just data from the service in services1 @@ -877,27 +548,147 @@ def test_list_services_merged(self, multi_backend_connection, config, backend1, enabled=True, configuration={"version": "0.5.8"}, attributes={}, - title="Test service", + title="Test WMTS service", created=datetime(2020, 4, 9, 15, 5, 8) ) services2 = {"services": [serv_metadata_wmts_foo.prepare_for_json()], "links": []} - test_user_id = "fakeuser" requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + test_user_id = "fakeuser" actual_services = implementation.list_services(user_id=test_user_id) # Construct expected result. We have get just data from the service in # services1 (there is only one) for conversion to a ServiceMetadata. - # For now we still ignore the key "links" in the outer dictionary. + # TODO: do we need to take care of the links part in the JSON as well? service1 = services1["services"][0] service1_md = ServiceMetadata.from_dict(service1) - service2 = services2["services"][0] - service2_md = ServiceMetadata.from_dict(service2) - expected_services = [service1_md, service2_md] + expected_services = [service1_md, serv_metadata_wmts_foo] - assert actual_services == expected_services + assert sorted(actual_services) == sorted(expected_services) + + def test_list_services_merged_multiple(self, multi_backend_connection, config, backend1, backend2, requests_mock): + services1 = self.TEST_SERVICES + serv_metadata_wmts_foo = ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMTS service", + created=datetime(2020, 4, 9, 15, 5, 8) + ) + serv_metadata_wms_bar = ServiceMetadata( + id="wms-bar", + process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, + url='https://oeo.net/wms/bar', + type="WMS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMS service", + created=datetime(2022, 2, 1, 13, 30, 3) + ) + services2 = {"services": [ + serv_metadata_wmts_foo.prepare_for_json(), + serv_metadata_wms_bar.prepare_for_json() + ], + "links": [] + } + requests_mock.get(backend1 + "/services", json=services1) + requests_mock.get(backend2 + "/services", json=services2) + implementation = AggregatorBackendImplementation( + backends=multi_backend_connection, config=config + ) + + test_user_id = "fakeuser" + actual_services = implementation.list_services(user_id=test_user_id) + + # Construct expected result. We have get just data from the service in + # services1 (there is only one) for conversion to a ServiceMetadata. + # TODO: do we need to take care of the links part in the JSON as well? + service1 = services1["services"][0] + service1_md = ServiceMetadata.from_dict(service1) + expected_services = [ + service1_md, serv_metadata_wmts_foo, serv_metadata_wms_bar + ] + + assert sorted(actual_services) == sorted(expected_services) + + def test_service_info(self, multi_backend_connection, config, backend1, backend2, requests_mock): + service1 = ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMTS service", + created=datetime(2020, 4, 9, 15, 5, 8) + ) + service2 = ServiceMetadata( + id="wms-bar", + process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, + url='https://oeo.net/wms/bar', + type="WMS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMS service", + created=datetime(2022, 2, 1, 13, 30, 3) + ) + requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) + + implementation = AggregatorBackendImplementation( + backends=multi_backend_connection, config=config + ) + + test_user_id = "fakeuser" + actual_service1 = implementation.service_info(user_id=test_user_id, service_id="wmts-foo") + assert actual_service1 == service1 + + actual_service2 = implementation.service_info(user_id=test_user_id, + service_id="wms-bar") + assert actual_service2 == service2 + + def test_service_info_not_found(self, multi_backend_connection, config, backend1, backend2, requests_mock): + service1 = ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMTS service", + created=datetime(2020, 4, 9, 15, 5, 8) + ) + service2 = ServiceMetadata( + id="wms-bar", + process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, + url='https://oeo.net/wms/bar', + type="WMS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMS service", + created=datetime(2022, 2, 1, 13, 30, 3) + ) + requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) + + implementation = AggregatorBackendImplementation( + backends=multi_backend_connection, config=config + ) + + test_user_id = "fakeuser" + with pytest.raises(ServiceNotFoundException): + actual_service1 = implementation.service_info(user_id=test_user_id, service_id="doesnotexist") class TestInternalCollectionMetadata: From 203ca218bb8e0c14d6bcbbeea9ca01707afbe896 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Fri, 4 Nov 2022 20:08:40 +0100 Subject: [PATCH 05/26] Issue #78 Implementing create_service - POST /services/ --- src/openeo_aggregator/backend.py | 34 ++++++++++++++++- tests/conftest.py | 9 +++++ tests/test_backend.py | 65 ++++++++++++++++++++++++-------- tests/test_views.py | 64 +++++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 17 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index b5b6e9de..0fcbcf3b 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -781,9 +781,11 @@ class AggregatorSecondaryServices(SecondaryServices): def __init__( self, backends: MultiBackendConnection, + processing: AggregatorProcessing ): super(AggregatorSecondaryServices, self).__init__() self._backends = backends + self._processing = processing def service_types(self) -> dict: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-service-types""" @@ -848,6 +850,31 @@ def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: raise ServiceNotFoundException(service_id) + def create_service(self, user_id: str, process_graph: dict, service_type: str, api_version: str, + configuration: dict) -> str: + """ + https://openeo.org/documentation/1.0/developers/api/reference.html#operation/create-service + :return: (location, openeo_identifier) + """ + + backend_id = self._processing.get_backend_for_process_graph( + process_graph=process_graph, api_version=api_version + ) + process_graph = self._processing.preprocess_process_graph(process_graph, backend_id=backend_id) + + con = self._backends.get_connection(backend_id) + try: + service = con.create_service(graph=process_graph, type=service_type) + except OpenEoApiError as e: + for exc_class in [ProcessGraphMissingException, ProcessGraphInvalidException]: + if e.code == exc_class.code: + raise exc_class + raise OpenEOApiException(f"Failed to create secondary service on backend {backend_id!r}: {e!r}") + except (OpenEoRestError, OpenEoClientException) as e: + raise OpenEOApiException(f"Failed to create secondary service on backend {backend_id!r}: {e!r}") + + return service.service_id + class AggregatorBackendImplementation(OpenEoBackendImplementation): # No basic auth: OIDC auth is required (to get EGI Check-in eduperson_entitlement data) @@ -875,7 +902,7 @@ def __init__(self, backends: MultiBackendConnection, config: AggregatorConfig): partitioned_job_tracker=partitioned_job_tracker ) - secondary_services = AggregatorSecondaryServices(backends=backends) + secondary_services = AggregatorSecondaryServices(backends=backends, processing=processing) super().__init__( catalog=catalog, @@ -1042,3 +1069,8 @@ def list_services(self, user_id: str) -> List[ServiceMetadata]: def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: return self.secondary_services.service_info(user_id=user_id, service_id=service_id) + + def create_service(self, user_id: str, process_graph: dict, service_type: str, api_version: str, + configuration: dict) -> Tuple[str, str]: + return self.secondary_services.create_service(user_id=user_id, process_graph=process_graph, + service_type=service_type, api_version=api_version, configuration=configuration) diff --git a/tests/conftest.py b/tests/conftest.py index 424c52c4..5ce45fee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -136,10 +136,19 @@ def backend_implementation(flask_app) -> AggregatorBackendImplementation: return flask_app.config["OPENEO_BACKEND_IMPLEMENTATION"] +def get_api040(flask_app: flask.Flask) -> ApiTester: + return ApiTester(api_version="0.4.0", client=flask_app.test_client()) + + def get_api100(flask_app: flask.Flask) -> ApiTester: return ApiTester(api_version="1.0.0", client=flask_app.test_client()) +@pytest.fixture +def api040(flask_app: flask.Flask) -> ApiTester: + return get_api040(flask_app) + + @pytest.fixture def api100(flask_app: flask.Flask) -> ApiTester: return get_api100(flask_app) diff --git a/tests/test_backend.py b/tests/test_backend.py index 9f8d7ae0..abf4ddea 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -15,6 +15,9 @@ from .conftest import DEFAULT_MEMOIZER_CONFIG +TEST_USER = "Mr.Test" + + class TestAggregatorBackendImplementation: def test_oidc_providers(self, multi_backend_connection, config, backend1, backend2, requests_mock): @@ -185,6 +188,7 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, expected.update(service_2) assert service_types == expected + # TODO: eliminate TEST_SERVICES (too long) when I find a better way to set up the test. TEST_SERVICES = { "services": [{ "id": "wms-a3cca9", @@ -521,14 +525,15 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, } def test_list_services_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): + """Given 2 backends where only 1 has 1 service and the other is empty, it lists that 1 service.""" + services1 = self.TEST_SERVICES services2 = {} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - test_user_id = "fakeuser" - actual_services = implementation.list_services(user_id=test_user_id) + actual_services = implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in services1 # (there is only one) for conversion to a ServiceMetadata. @@ -539,6 +544,8 @@ def test_list_services_simple(self, multi_backend_connection, config, backend1, assert actual_services == expected_services def test_list_services_merged(self, multi_backend_connection, config, backend1, backend2, requests_mock): + """Given 2 backends with each 1 service, it lists both services.""" + services1 = self.TEST_SERVICES serv_metadata_wmts_foo = ServiceMetadata( id="wmts-foo", @@ -556,8 +563,7 @@ def test_list_services_merged(self, multi_backend_connection, config, backend1, requests_mock.get(backend2 + "/services", json=services2) implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - test_user_id = "fakeuser" - actual_services = implementation.list_services(user_id=test_user_id) + actual_services = implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in # services1 (there is only one) for conversion to a ServiceMetadata. @@ -565,10 +571,11 @@ def test_list_services_merged(self, multi_backend_connection, config, backend1, service1 = services1["services"][0] service1_md = ServiceMetadata.from_dict(service1) expected_services = [service1_md, serv_metadata_wmts_foo] - assert sorted(actual_services) == sorted(expected_services) def test_list_services_merged_multiple(self, multi_backend_connection, config, backend1, backend2, requests_mock): + """Given multiple services in 2 backends, it lists all services from all backends.""" + services1 = self.TEST_SERVICES serv_metadata_wmts_foo = ServiceMetadata( id="wmts-foo", @@ -604,8 +611,7 @@ def test_list_services_merged_multiple(self, multi_backend_connection, config, b backends=multi_backend_connection, config=config ) - test_user_id = "fakeuser" - actual_services = implementation.list_services(user_id=test_user_id) + actual_services = implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in # services1 (there is only one) for conversion to a ServiceMetadata. @@ -619,6 +625,8 @@ def test_list_services_merged_multiple(self, multi_backend_connection, config, b assert sorted(actual_services) == sorted(expected_services) def test_service_info(self, multi_backend_connection, config, backend1, backend2, requests_mock): + """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + service1 = ServiceMetadata( id="wmts-foo", process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, @@ -643,20 +651,19 @@ def test_service_info(self, multi_backend_connection, config, backend1, backend2 ) requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) - implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - test_user_id = "fakeuser" - actual_service1 = implementation.service_info(user_id=test_user_id, service_id="wmts-foo") + actual_service1 = implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") assert actual_service1 == service1 - actual_service2 = implementation.service_info(user_id=test_user_id, - service_id="wms-bar") + actual_service2 = implementation.service_info(user_id=TEST_USER, service_id="wms-bar") assert actual_service2 == service2 - def test_service_info_not_found(self, multi_backend_connection, config, backend1, backend2, requests_mock): + def test_service_info_wrong_id(self, multi_backend_connection, config, backend1, backend2, requests_mock): + """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" + service1 = ServiceMetadata( id="wmts-foo", process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, @@ -681,14 +688,40 @@ def test_service_info_not_found(self, multi_backend_connection, config, backend1 ) requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) - implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - test_user_id = "fakeuser" with pytest.raises(ServiceNotFoundException): - actual_service1 = implementation.service_info(user_id=test_user_id, service_id="doesnotexist") + _ = implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") + + @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) + def test_create_service(self, multi_backend_connection, config, backend1, requests_mock, api_version): + """When it gets a correct params for a new service, it succesfully create it.""" + + # Set up responses for creating the service in backend 1 + expected_openeo_id = "wmts-foo" + expected_location = backend1 + "/services/wmts-foo" + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + requests_mock.post( + backend1 + "/services", + headers={ + "OpenEO-Identifier": expected_openeo_id, + "Location": expected_location + }, + status_code=201) + + implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + actual_openeo_id = implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_version, + configuration={} + ) + + assert actual_openeo_id == expected_openeo_id class TestInternalCollectionMetadata: diff --git a/tests/test_views.py b/tests/test_views.py index 829b0ff8..ce654d97 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1182,6 +1182,70 @@ def post_jobs(request: requests.Request, context): }}} ] +class TestSecondaryServices: + + # TODO: add view tests for list service types, list_services, servicfe_info + + def test_create_wmts_040(self, api040, requests_mock, backend1): + api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + + expected_openeo_id = 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' + expected_location = "/openeo/0.4.0/services/" + expected_openeo_id + + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + # The process_graph/process format is slightly different between api v0.4 and v1.0 + post_data = { + "type": 'WMTS', + "process_graph": process_graph, + "custom_param": 45, + "title": "My Service", + "description": "Service description" + } + + requests_mock.post( + backend1 + "/services", + headers={ + "OpenEO-Identifier": expected_openeo_id, + "Location": expected_location + }, + status_code=201) + + resp = api040.post('/services', json=post_data).assert_status_code(201) + assert resp.headers['OpenEO-Identifier'] == expected_openeo_id + assert resp.headers['Location'] == expected_location + + def test_create_wmts_100(self, api100, requests_mock, backend1): + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + + # used both to set up data and to validate at the end + expected_openeo_id = 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' + expected_location = "/openeo/1.0.0/services/" + expected_openeo_id + + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + # The process_graph/process format is slightly different between api v0.4 and v1.0 + post_data = { + "type": 'WMTS', + "process": { + "process_graph": process_graph, + "id": "filter_temporal_wmts" + }, + "custom_param": 45, + "title": "My Service", + "description": "Service description" + } + requests_mock.post( + backend1 + "/services", + headers={ + "OpenEO-Identifier": expected_openeo_id, + "Location": expected_location + }, + status_code=201) + + resp = api100.post('/services', json=post_data).assert_status_code(201) + + assert resp.headers['OpenEO-Identifier'] == 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' + assert resp.headers['Location'] == expected_location + class TestResilience: From e78e6d635e5c75fef4fdda7dffc4cc1333914633 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 7 Nov 2022 10:20:18 +0100 Subject: [PATCH 06/26] Issue #78 Adding tests for views for SecondaryServices --- src/openeo_aggregator/backend.py | 4 +- tests/conftest.py | 19 +++++++ tests/test_backend.py | 97 +++++++++++++++++++++++++------- tests/test_views.py | 69 +++++++++++++++++++++++ 4 files changed, 168 insertions(+), 21 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 0fcbcf3b..fdc1a5a0 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -854,7 +854,6 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a configuration: dict) -> str: """ https://openeo.org/documentation/1.0/developers/api/reference.html#operation/create-service - :return: (location, openeo_identifier) """ backend_id = self._processing.get_backend_for_process_graph( @@ -864,7 +863,10 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a con = self._backends.get_connection(backend_id) try: + # create_service can raise ServiceUnsupportedException and OpenEOApiException. service = con.create_service(graph=process_graph, type=service_type) + + # TODO: This exception handling was copy-pasted. What do we actually need here? except OpenEoApiError as e: for exc_class in [ProcessGraphMissingException, ProcessGraphInvalidException]: if e.code == exc_class.code: diff --git a/tests/conftest.py b/tests/conftest.py index 5ce45fee..04aaf2b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -136,6 +136,16 @@ def backend_implementation(flask_app) -> AggregatorBackendImplementation: return flask_app.config["OPENEO_BACKEND_IMPLEMENTATION"] +@pytest.fixture(params=["0.4.0", "1.0.0"]) +def api_version(request): + """To go through all relevant API versions""" + return request.param + + +def get_api_version(flask_app, api_version) -> ApiTester: + return ApiTester(api_version=api_version, client=flask_app.test_client()) + + def get_api040(flask_app: flask.Flask) -> ApiTester: return ApiTester(api_version="0.4.0", client=flask_app.test_client()) @@ -144,6 +154,15 @@ def get_api100(flask_app: flask.Flask) -> ApiTester: return ApiTester(api_version="1.0.0", client=flask_app.test_client()) +@pytest.fixture +def api(flask_app, api_version) -> ApiTester: + """Get an ApiTester for each version. + + Useful when it easy to test several API versions with (mostly) the same test code. + But when the difference is too big, just write separate tests. + """ + return get_api_version(flask_app, api_version) + @pytest.fixture def api040(flask_app: flask.Flask) -> ApiTester: return get_api040(flask_app) diff --git a/tests/test_backend.py b/tests/test_backend.py index abf4ddea..e1811648 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -142,10 +142,10 @@ def test_service_types_simple(self, multi_backend_connection, config, backend1, } requests_mock.get(backend1 + "/service_types", json=single_service_type) requests_mock.get(backend2 + "/service_types", json=single_service_type) - implementation = AggregatorBackendImplementation( + abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - service_types = implementation.service_types() + service_types = abe_implementation.service_types() assert service_types == single_service_type def test_service_types_merging(self, multi_backend_connection, config, backend1, backend2, requests_mock): @@ -180,13 +180,15 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, } requests_mock.get(backend1 + "/service_types", json=service_1) requests_mock.get(backend2 + "/service_types", json=service_2) - implementation = AggregatorBackendImplementation( + abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - service_types = implementation.service_types() - expected = dict(service_1) - expected.update(service_2) - assert service_types == expected + + actual_service_types = abe_implementation.service_types() + + expected_service_types = dict(service_1) + expected_service_types.update(service_2) + assert actual_service_types == expected_service_types # TODO: eliminate TEST_SERVICES (too long) when I find a better way to set up the test. TEST_SERVICES = { @@ -531,9 +533,9 @@ def test_list_services_simple(self, multi_backend_connection, config, backend1, services2 = {} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) - implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - actual_services = implementation.list_services(user_id=TEST_USER) + actual_services = abe_implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in services1 # (there is only one) for conversion to a ServiceMetadata. @@ -561,9 +563,9 @@ def test_list_services_merged(self, multi_backend_connection, config, backend1, services2 = {"services": [serv_metadata_wmts_foo.prepare_for_json()], "links": []} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) - implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - actual_services = implementation.list_services(user_id=TEST_USER) + actual_services = abe_implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in # services1 (there is only one) for conversion to a ServiceMetadata. @@ -607,11 +609,11 @@ def test_list_services_merged_multiple(self, multi_backend_connection, config, b } requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) - implementation = AggregatorBackendImplementation( + abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - actual_services = implementation.list_services(user_id=TEST_USER) + actual_services = abe_implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in # services1 (there is only one) for conversion to a ServiceMetadata. @@ -651,14 +653,14 @@ def test_service_info(self, multi_backend_connection, config, backend1, backend2 ) requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) - implementation = AggregatorBackendImplementation( + abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - actual_service1 = implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") + actual_service1 = abe_implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") assert actual_service1 == service1 - actual_service2 = implementation.service_info(user_id=TEST_USER, service_id="wms-bar") + actual_service2 = abe_implementation.service_info(user_id=TEST_USER, service_id="wms-bar") assert actual_service2 == service2 def test_service_info_wrong_id(self, multi_backend_connection, config, backend1, backend2, requests_mock): @@ -688,12 +690,12 @@ def test_service_info_wrong_id(self, multi_backend_connection, config, backend1, ) requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) - implementation = AggregatorBackendImplementation( + abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) with pytest.raises(ServiceNotFoundException): - _ = implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") + _ = abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) def test_create_service(self, multi_backend_connection, config, backend1, requests_mock, api_version): @@ -711,9 +713,9 @@ def test_create_service(self, multi_backend_connection, config, backend1, reques }, status_code=201) - implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - actual_openeo_id = implementation.create_service( + actual_openeo_id = abe_implementation.create_service( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", @@ -723,6 +725,61 @@ def test_create_service(self, multi_backend_connection, config, backend1, reques assert actual_openeo_id == expected_openeo_id + # TODO: test a failing case for test_create_service + @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) + def test_create_service_backend_returns_error( + self, multi_backend_connection, config, backend1, requests_mock, api_version + ): + from openeo.rest import OpenEoApiError, OpenEoRestError + # TODO: Two exception types should be re-raised: ProcessGraphMissingException, ProcessGraphInvalidException + + for exc_class in [OpenEoApiError, OpenEoRestError]: + # Set up responses for creating the service in backend 1 + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + requests_mock.post( + backend1 + "/services", + exc=exc_class("Some server error"), + ) + + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + with pytest.raises(OpenEOApiException): + _ = abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_version, + configuration={} + ) + + @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) + def test_create_service_backend_reraises( + self, multi_backend_connection, config, backend1, requests_mock, api_version + ): + from openeo_driver.errors import ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException + + for exc_class in [ProcessGraphMissingException, + ProcessGraphInvalidException, + ServiceUnsupportedException]: + # Set up responses for creating the service in backend 1 + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + requests_mock.post( + backend1 + "/services", + exc=exc_class("Some server error"), + ) + + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + # These exception types should be re-raised, not become an OpenEOApiException. + with pytest.raises(exc_class): + _ = abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_version, + configuration={} + ) + class TestInternalCollectionMetadata: diff --git a/tests/test_views.py b/tests/test_views.py index ce654d97..eb343a31 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1182,10 +1182,79 @@ def post_jobs(request: requests.Request, context): }}} ] + class TestSecondaryServices: # TODO: add view tests for list service types, list_services, servicfe_info + def test_service_types_simple(self, api, backend1, backend2, requests_mock): + single_service_type = { + "WMTS": { + "configuration": { + "colormap": { + "default": "YlGn", + "description": + "The colormap to apply to single band layers", + "type": "string" + }, + "version": { + "default": "1.0.0", + "description": "The WMTS version to use.", + "enum": ["1.0.0"], + "type": "string" + } + }, + "links": [], + "process_parameters": [], + "title": "Web Map Tile Service" + } + } + requests_mock.get(backend1 + "/service_types", json=single_service_type) + requests_mock.get(backend2 + "/service_types", json=single_service_type) + + resp = api.get('/service_types').assert_status_code(200) + assert resp.json == single_service_type + + def test_service_types_merging(self, api, backend1, backend2, requests_mock): + service_type_1 = { + "WMTS": { + "configuration": { + "colormap": { + "default": "YlGn", + "description": + "The colormap to apply to single band layers", + "type": "string" + }, + "version": { + "default": "1.0.0", + "description": "The WMTS version to use.", + "enum": ["1.0.0"], + "type": "string" + } + }, + "links": [], + "process_parameters": [], + "title": "Web Map Tile Service" + } + } + service_type_2 = { + "WMS": { + "title": "OGC Web Map Service", + "configuration": {}, + "process_parameters": [], + "links": [] + } + } + requests_mock.get(backend1 + "/service_types", json=service_type_1) + requests_mock.get(backend2 + "/service_types", json=service_type_2) + + resp = api.get("/service_types").assert_status_code(200) + actual_service_types = resp.json + + expected_service_types = dict(service_type_1) + expected_service_types.update(service_type_2) + assert actual_service_types == expected_service_types + def test_create_wmts_040(self, api040, requests_mock, backend1): api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) From 38f48f90466d2ffaccc6f7acde3e575051b643e5 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 7 Nov 2022 14:38:31 +0100 Subject: [PATCH 07/26] Issue #78 Cleaned up the new tests a bit. --- tests/test_backend.py | 577 +++++++++--------------------------------- tests/test_views.py | 113 ++++++++- 2 files changed, 228 insertions(+), 462 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index e1811648..3b4ecced 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -12,6 +12,8 @@ ServiceNotFoundException from openeo_driver.testing import DictSubSet from openeo_driver.users.oidc import OidcProvider +from openeo_driver.errors import ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException +from openeo.rest import OpenEoApiError, OpenEoRestError from .conftest import DEFAULT_MEMOIZER_CONFIG @@ -119,6 +121,9 @@ def test_file_formats_merging(self, multi_backend_connection, config, backend1, } def test_service_types_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): + """Given 2 backends and only 1 backend has a single service type, then the aggregator + returns that 1 service type's metadata. + """ single_service_type = { "WMTS": { "configuration": { @@ -149,7 +154,8 @@ def test_service_types_simple(self, multi_backend_connection, config, backend1, assert service_types == single_service_type def test_service_types_merging(self, multi_backend_connection, config, backend1, backend2, requests_mock): - service_1 = { + """Given 2 backends with each 1 service type, then the aggregator lists both service types.""" + service_type_1 = { "WMTS": { "configuration": { "colormap": { @@ -170,7 +176,7 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, "title": "Web Map Tile Service" } } - service_2 = { + service_type_2 = { "WMS": { "title": "OGC Web Map Service", "configuration": {}, @@ -178,358 +184,54 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, "links": [] } } - requests_mock.get(backend1 + "/service_types", json=service_1) - requests_mock.get(backend2 + "/service_types", json=service_2) + requests_mock.get(backend1 + "/service_types", json=service_type_1) + requests_mock.get(backend2 + "/service_types", json=service_type_2) abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) actual_service_types = abe_implementation.service_types() - expected_service_types = dict(service_1) - expected_service_types.update(service_2) + expected_service_types = dict(service_type_1) + expected_service_types.update(service_type_2) assert actual_service_types == expected_service_types - # TODO: eliminate TEST_SERVICES (too long) when I find a better way to set up the test. - TEST_SERVICES = { - "services": [{ - "id": "wms-a3cca9", - "title": "NDVI based on Sentinel 2", - "description": "Deriving minimum NDVI measurements over pixel time series of Sentinel 2", - "url": "https://example.openeo.org/wms/wms-a3cca9", - "type": "wms", - "enabled": True, - "process": { - "id": "ndvi", - "summary": "string", - "description": "string", - "parameters": [{ - "schema": { - "parameters": [{ - "schema": { - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - }, - "name": "string", - "description": "string", - "optional": False, - "deprecated": False, - "experimental": False, - "default": None - }], - "returns": { - "description": "string", - "schema": { - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - }, - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - }, - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - }, - "name": "string", - "description": "string", - "optional": False, - "deprecated": False, - "experimental": False, - "default": None - }], - "returns": { - "description": "string", - "schema": { - "type": "array", - "subtype": "string", - "pattern": "/regex/", - "enum": [None], - "minimum": 0, - "maximum": 0, - "minItems": 0, - "maxItems": 0, - "items": [{}], - "deprecated": False - } - }, - "categories": ["string"], - "deprecated": False, - "experimental": False, - "exceptions": { - "Error Code1": { - "description": "string", - "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", - "http": 400 - }, - "Error Code2": { - "description": "string", - "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", - "http": 400 - } - }, - "examples": [{ - "title": "string", - "description": "string", - "arguments": { - "property1": { - "from_parameter": None, - "from_node": None, - "process_graph": None - }, - "property2": { - "from_parameter": None, - "from_node": None, - "process_graph": None - } - }, - "returns": None - }], - "links": [{ - "rel": "related", - "href": "https://example.openeo.org", - "type": "text/html", - "title": "openEO" - }], - "process_graph": { - "dc": { - "process_id": "load_collection", - "arguments": { - "id": "Sentinel-2", - "spatial_extent": { - "west": 16.1, - "east": 16.6, - "north": 48.6, - "south": 47.2 - }, - "temporal_extent": ["2018-01-01", "2018-02-01"] - } - }, - "bands": { - "process_id": "filter_bands", - "description": - "Filter and order the bands. The order is important for the following reduce operation.", - "arguments": { - "data": { - "from_node": "dc" - }, - "bands": ["B08", "B04", "B02"] - } - }, - "evi": { - "process_id": "reduce", - "description": - "Compute the EVI. Formula: 2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)", - "arguments": { - "data": { - "from_node": "bands" - }, - "dimension": "bands", - "reducer": { - "process_graph": { - "nir": { - "process_id": "array_element", - "arguments": { - "data": { - "from_parameter": "data" - }, - "index": 0 - } - }, - "red": { - "process_id": "array_element", - "arguments": { - "data": { - "from_parameter": "data" - }, - "index": 1 - } - }, - "blue": { - "process_id": "array_element", - "arguments": { - "data": { - "from_parameter": "data" - }, - "index": 2 - } - }, - "sub": { - "process_id": "subtract", - "arguments": { - "data": [{ - "from_node": "nir" - }, { - "from_node": "red" - }] - } - }, - "p1": { - "process_id": "product", - "arguments": { - "data": [6, { - "from_node": "red" - }] - } - }, - "p2": { - "process_id": "product", - "arguments": { - "data": - [-7.5, { - "from_node": "blue" - }] - } - }, - "sum": { - "process_id": "sum", - "arguments": { - "data": [ - 1, { - "from_node": "nir" - }, { - "from_node": "p1" - }, { - "from_node": "p2" - } - ] - } - }, - "div": { - "process_id": "divide", - "arguments": { - "data": [{ - "from_node": "sub" - }, { - "from_node": "sum" - }] - } - }, - "p3": { - "process_id": "product", - "arguments": { - "data": - [2.5, { - "from_node": "div" - }] - }, - "result": True - } - } - } - } - }, - "mintime": { - "process_id": "reduce", - "description": - "Compute a minimum time composite by reducing the temporal dimension", - "arguments": { - "data": { - "from_node": "evi" - }, - "dimension": "temporal", - "reducer": { - "process_graph": { - "min": { - "process_id": "min", - "arguments": { - "data": { - "from_parameter": "data" - } - }, - "result": True - } - } - } - } - }, - "save": { - "process_id": "save_result", - "arguments": { - "data": { - "from_node": "mintime" - }, - "format": "GTiff" - }, - "result": True - } - } - }, - "configuration": { - "version": "1.3.0" - }, - "attributes": { - "layers": ["ndvi", "evi"] - }, - "created": "2017-01-01T09:32:12Z", - "plan": "free", - "costs": 12.98, - "budget": 100, - "usage": { - "cpu": { - "value": 40668, - "unit": "cpu-seconds" - }, - "duration": { - "value": 2611, - "unit": "seconds" - }, - "memory": { - "value": 108138811, - "unit": "mb-seconds" - }, - "network": { - "value": 0, - "unit": "kb" - }, - "storage": { - "value": 55, - "unit": "mb" - } - } - }], - "links": [{ - "rel": "related", - "href": "https://example.openeo.org", - "type": "text/html", - "title": "openEO" - }] - } - - def test_list_services_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): - """Given 2 backends where only 1 has 1 service and the other is empty, it lists that 1 service.""" - - services1 = self.TEST_SERVICES + @pytest.fixture + def service_metadata_wmts_foo(self): + return ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMTS service", + created=datetime(2020, 4, 9, 15, 5, 8) + ) + + @pytest.fixture + def service_metadata_wms_bar(self): + return ServiceMetadata( + id="wms-bar", + process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, + url='https://oeo.net/wms/bar', + type="WMS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMS service", + created=datetime(2022, 2, 1, 13, 30, 3) + ) + + def test_list_services_simple( + self, multi_backend_connection, config, backend1, backend2, requests_mock, + service_metadata_wmts_foo + ): + """Given 2 backends but only 1 backend has a single service, then the aggregator + returns that 1 service's metadata. + """ + services1 = {"services": [service_metadata_wmts_foo.prepare_for_json()], "links": []} services2 = {} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) @@ -545,65 +247,65 @@ def test_list_services_simple(self, multi_backend_connection, config, backend1, ] assert actual_services == expected_services - def test_list_services_merged(self, multi_backend_connection, config, backend1, backend2, requests_mock): - """Given 2 backends with each 1 service, it lists both services.""" - - services1 = self.TEST_SERVICES - serv_metadata_wmts_foo = ServiceMetadata( - id="wmts-foo", - process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, - url='https://oeo.net/wmts/foo', - type="WMTS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMTS service", - created=datetime(2020, 4, 9, 15, 5, 8) - ) - services2 = {"services": [serv_metadata_wmts_foo.prepare_for_json()], "links": []} + def test_list_services_merged( + self, multi_backend_connection, config, backend1, backend2, requests_mock, + service_metadata_wmts_foo, service_metadata_wms_bar + ): + """Given 2 backends with each 1 service, then the aggregator lists both services.""" + services1 = {"services": [service_metadata_wmts_foo.prepare_for_json()], "links": []} + services2 = {"services": [service_metadata_wms_bar.prepare_for_json()], "links": []} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) actual_services = abe_implementation.list_services(user_id=TEST_USER) - # Construct expected result. We have get just data from the service in - # services1 (there is only one) for conversion to a ServiceMetadata. - # TODO: do we need to take care of the links part in the JSON as well? - service1 = services1["services"][0] - service1_md = ServiceMetadata.from_dict(service1) - expected_services = [service1_md, serv_metadata_wmts_foo] + expected_services = [service_metadata_wmts_foo, service_metadata_wms_bar] assert sorted(actual_services) == sorted(expected_services) - def test_list_services_merged_multiple(self, multi_backend_connection, config, backend1, backend2, requests_mock): - """Given multiple services in 2 backends, it lists all services from all backends.""" - - services1 = self.TEST_SERVICES - serv_metadata_wmts_foo = ServiceMetadata( - id="wmts-foo", - process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, - url='https://oeo.net/wmts/foo', - type="WMTS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMTS service", - created=datetime(2020, 4, 9, 15, 5, 8) - ) - serv_metadata_wms_bar = ServiceMetadata( - id="wms-bar", - process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, - url='https://oeo.net/wms/bar', - type="WMS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMS service", - created=datetime(2022, 2, 1, 13, 30, 3) - ) + def test_list_services_merged_multiple( + self, multi_backend_connection, config, backend1, backend2, requests_mock, + service_metadata_wmts_foo, service_metadata_wms_bar + ): + """Given multiple services across 2 backends, the aggregator lists all service types from all backends.""" + services1 = { + "services": [{ + "id": "wms-nvdi", + "title": "NDVI based on Sentinel 2", + "description": "Deriving minimum NDVI measurements over pixel time series of Sentinel 2", + "url": "https://example.openeo.org/wms/wms-nvdi", + "type": "wms", + "enabled": True, + "process": { + "id": "ndvi", + "summary": "string", + "description": "string", + "links": [{ + "rel": "related", + "href": "https://example.openeo.org", + "type": "text/html", + "title": "openEO" + }], + "process_graph": {"foo": {"process_id": "foo", "arguments": {}}}, + }, + "configuration": { + "version": "1.3.0" + }, + "attributes": { + "layers": ["ndvi", "evi"] + }, + "created": "2017-01-01T09:32:12Z", + }], + "links": [{ + "rel": "related", + "href": "https://example.openeo.org", + "type": "text/html", + "title": "openEO" + }] + } services2 = {"services": [ - serv_metadata_wmts_foo.prepare_for_json(), - serv_metadata_wms_bar.prepare_for_json() + service_metadata_wmts_foo.prepare_for_json(), + service_metadata_wms_bar.prepare_for_json() ], "links": [] } @@ -621,75 +323,36 @@ def test_list_services_merged_multiple(self, multi_backend_connection, config, b service1 = services1["services"][0] service1_md = ServiceMetadata.from_dict(service1) expected_services = [ - service1_md, serv_metadata_wmts_foo, serv_metadata_wms_bar + service1_md, service_metadata_wmts_foo, service_metadata_wms_bar ] assert sorted(actual_services) == sorted(expected_services) - def test_service_info(self, multi_backend_connection, config, backend1, backend2, requests_mock): + def test_service_info( + self, multi_backend_connection, config, backend1, backend2, requests_mock, + service_metadata_wmts_foo, service_metadata_wms_bar + ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" - - service1 = ServiceMetadata( - id="wmts-foo", - process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, - url='https://oeo.net/wmts/foo', - type="WMTS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMTS service", - created=datetime(2020, 4, 9, 15, 5, 8) - ) - service2 = ServiceMetadata( - id="wms-bar", - process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, - url='https://oeo.net/wms/bar', - type="WMS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMS service", - created=datetime(2022, 2, 1, 13, 30, 3) - ) - requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) + requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) + # Check the expected metadata on *both* of the services. actual_service1 = abe_implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") - assert actual_service1 == service1 - + assert actual_service1 == service_metadata_wmts_foo actual_service2 = abe_implementation.service_info(user_id=TEST_USER, service_id="wms-bar") - assert actual_service2 == service2 + assert actual_service2 == service_metadata_wms_bar - def test_service_info_wrong_id(self, multi_backend_connection, config, backend1, backend2, requests_mock): + def test_service_info_wrong_id( + self, multi_backend_connection, config, backend1, backend2, requests_mock, + service_metadata_wmts_foo, service_metadata_wms_bar + ): """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" - service1 = ServiceMetadata( - id="wmts-foo", - process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, - url='https://oeo.net/wmts/foo', - type="WMTS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMTS service", - created=datetime(2020, 4, 9, 15, 5, 8) - ) - service2 = ServiceMetadata( - id="wms-bar", - process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, - url='https://oeo.net/wms/bar', - type="WMS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMS service", - created=datetime(2022, 2, 1, 13, 30, 3) - ) - requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) + requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) @@ -699,7 +362,7 @@ def test_service_info_wrong_id(self, multi_backend_connection, config, backend1, @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) def test_create_service(self, multi_backend_connection, config, backend1, requests_mock, api_version): - """When it gets a correct params for a new service, it succesfully create it.""" + """When it gets a correct params for a new service, it succesfully creates it.""" # Set up responses for creating the service in backend 1 expected_openeo_id = "wmts-foo" @@ -725,16 +388,13 @@ def test_create_service(self, multi_backend_connection, config, backend1, reques assert actual_openeo_id == expected_openeo_id - # TODO: test a failing case for test_create_service @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) def test_create_service_backend_returns_error( self, multi_backend_connection, config, backend1, requests_mock, api_version ): - from openeo.rest import OpenEoApiError, OpenEoRestError - # TODO: Two exception types should be re-raised: ProcessGraphMissingException, ProcessGraphInvalidException - for exc_class in [OpenEoApiError, OpenEoRestError]: - # Set up responses for creating the service in backend 1 + # Set up responses for creating the service in backend 1: + # This time the backend raises an error, one that will be reported as a OpenEOApiException. process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", @@ -756,12 +416,11 @@ def test_create_service_backend_returns_error( def test_create_service_backend_reraises( self, multi_backend_connection, config, backend1, requests_mock, api_version ): - from openeo_driver.errors import ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException - for exc_class in [ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException]: # Set up responses for creating the service in backend 1 + # This time the backend raises an error, one that will simply be re-raised/passed on as it is. process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", diff --git a/tests/test_views.py b/tests/test_views.py index eb343a31..8d078f01 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,3 +1,4 @@ +from datetime import datetime import itertools import logging import re @@ -9,10 +10,10 @@ from openeo.rest.connection import url_join from openeo_aggregator.backend import AggregatorCollectionCatalog from openeo_aggregator.config import AggregatorConfig -from openeo_aggregator.connection import MultiBackendConnection from openeo_aggregator.testing import clock_mock from openeo_driver.errors import JobNotFoundException, JobNotFinishedException, \ ProcessGraphInvalidException +from openeo_driver.backend import ServiceMetadata from openeo_driver.testing import ApiTester, TEST_USER_AUTH_HEADER, TEST_USER, TEST_USER_BEARER_TOKEN, DictSubSet, \ RegexMatcher from .conftest import assert_dict_subset, get_api100, get_flask_app @@ -1188,6 +1189,9 @@ class TestSecondaryServices: # TODO: add view tests for list service types, list_services, servicfe_info def test_service_types_simple(self, api, backend1, backend2, requests_mock): + """Given 2 backends but only 1 backend has a single service, then the aggregator + returns that 1 service's metadata. + """ single_service_type = { "WMTS": { "configuration": { @@ -1216,6 +1220,7 @@ def test_service_types_simple(self, api, backend1, backend2, requests_mock): assert resp.json == single_service_type def test_service_types_merging(self, api, backend1, backend2, requests_mock): + """Given 2 backends with each 1 service, then the aggregator lists both services.""" service_type_1 = { "WMTS": { "configuration": { @@ -1255,6 +1260,106 @@ def test_service_types_merging(self, api, backend1, backend2, requests_mock): expected_service_types.update(service_type_2) assert actual_service_types == expected_service_types + def test_service_info_api100(self, api100, backend1, backend2, requests_mock): + """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + + service1 = ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMTS service", + ) + service2 = ServiceMetadata( + id="wms-bar", + process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, + url='https://oeo.net/wms/bar', + type="WMS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={"layers": ["ndvi", "evi"]}, + title="Test WMS service", + ) + requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) + api100.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) + + # Retrieve and verify the metadata for both services + resp = api100.get("/services/wmts-foo").assert_status_code(200) + actual_service1 = ServiceMetadata( + id=resp.json["id"], + process=resp.json["process"], + url=resp.json["url"], + type=resp.json["type"], + enabled=resp.json["enabled"], + configuration=resp.json["configuration"], + attributes=resp.json["attributes"], + title=resp.json["title"], + ) + assert actual_service1 == service1 + + resp = api100.get("/services/wms-bar").assert_status_code(200) + actual_service2 = ServiceMetadata( + id=resp.json["id"], + process=resp.json["process"], + url=resp.json["url"], + type=resp.json["type"], + enabled=resp.json["enabled"], + configuration=resp.json["configuration"], + attributes=resp.json["attributes"], + title=resp.json["title"], + ) + assert actual_service2 == service2 + + def test_service_info_api040(self, api040, backend1, backend2, requests_mock): + """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + + service1 = ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={"layers": ["ndvi", "evi"]}, + title="Test WMTS service", + ) + service2 = ServiceMetadata( + id="wms-bar", + process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, + url='https://oeo.net/wms/bar', + type="WMS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMS service", + ) + requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) + api040.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) + + # Retrieve and verify the metadata for both services + resp = api040.get("/services/wmts-foo").assert_status_code(200) + # required = ["id", "process", "url", "type", "enabled", "configuration", "attributes", "title"] + assert service1.id == resp.json["id"] + assert service1.process["process_graph"] == resp.json["process_graph"] + assert service1.url == resp.json["url"] + assert service1.type == resp.json["type"] + assert service1.title == resp.json["title"] + assert service1.attributes == resp.json["attributes"] + + resp = api040.get("/services/wms-bar").assert_status_code(200) + assert service2.id == resp.json["id"] + assert service2.process["process_graph"] == resp.json["process_graph"] + assert service2.url == resp.json["url"] + assert service2.type == resp.json["type"] + assert service2.title == resp.json["title"] + assert service2.attributes == resp.json["attributes"] + assert service2.attributes == resp.json["attributes"] + def test_create_wmts_040(self, api040, requests_mock, backend1): api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) @@ -1277,7 +1382,8 @@ def test_create_wmts_040(self, api040, requests_mock, backend1): "OpenEO-Identifier": expected_openeo_id, "Location": expected_location }, - status_code=201) + status_code=201 + ) resp = api040.post('/services', json=post_data).assert_status_code(201) assert resp.headers['OpenEO-Identifier'] == expected_openeo_id @@ -1308,7 +1414,8 @@ def test_create_wmts_100(self, api100, requests_mock, backend1): "OpenEO-Identifier": expected_openeo_id, "Location": expected_location }, - status_code=201) + status_code=201 + ) resp = api100.post('/services', json=post_data).assert_status_code(201) From 5b72f756f3bacf3e1d51ee17229a73daaa7ddcf5 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 7 Nov 2022 15:32:11 +0100 Subject: [PATCH 08/26] Issue #78 Adding more tests for service_info and create_service --- .flake8 | 8 +++ tests/test_backend.py | 83 ++++++++++++----------- tests/test_views.py | 154 ++++++++++++++++++++++++------------------ 3 files changed, 140 insertions(+), 105 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..9292294a --- /dev/null +++ b/.flake8 @@ -0,0 +1,8 @@ +[flake8] +# extend-ignore = E203 +exclude = + # No need to traverse our git directory + .git, + # There's no value in checking cache directories + __pycache__, +max-line-length=120 diff --git a/tests/test_backend.py b/tests/test_backend.py index 3b4ecced..675d7043 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -389,55 +389,56 @@ def test_create_service(self, multi_backend_connection, config, backend1, reques assert actual_openeo_id == expected_openeo_id @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) - def test_create_service_backend_returns_error( - self, multi_backend_connection, config, backend1, requests_mock, api_version + @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) + def test_create_service_backend_raises_openeoapiexception( + self, multi_backend_connection, config, backend1, requests_mock, api_version, exception_class ): - for exc_class in [OpenEoApiError, OpenEoRestError]: - # Set up responses for creating the service in backend 1: - # This time the backend raises an error, one that will be reported as a OpenEOApiException. - process_graph = {"foo": {"process_id": "foo", "arguments": {}}} - requests_mock.post( - backend1 + "/services", - exc=exc_class("Some server error"), - ) + # for exc_class in [OpenEoApiError, OpenEoRestError]: + # Set up responses for creating the service in backend 1: + # This time the backend raises an error, one that will be reported as a OpenEOApiException. + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + requests_mock.post( + backend1 + "/services", + exc=exception_class("Some server error"), + ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - with pytest.raises(OpenEOApiException): - _ = abe_implementation.create_service( - user_id=TEST_USER, - process_graph=process_graph, - service_type="WMTS", - api_version=api_version, - configuration={} - ) + with pytest.raises(OpenEOApiException): + _ = abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_version, + configuration={} + ) @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) + @pytest.mark.parametrize("exception_class", + [ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException] + ) def test_create_service_backend_reraises( - self, multi_backend_connection, config, backend1, requests_mock, api_version + self, multi_backend_connection, config, backend1, requests_mock, api_version, exception_class ): - for exc_class in [ProcessGraphMissingException, - ProcessGraphInvalidException, - ServiceUnsupportedException]: - # Set up responses for creating the service in backend 1 - # This time the backend raises an error, one that will simply be re-raised/passed on as it is. - process_graph = {"foo": {"process_id": "foo", "arguments": {}}} - requests_mock.post( - backend1 + "/services", - exc=exc_class("Some server error"), - ) + # Set up responses for creating the service in backend 1 + # This time the backend raises an error, one that will simply be re-raised/passed on as it is. + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + requests_mock.post( + backend1 + "/services", + exc=exception_class("Some server error"), + ) + + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - - # These exception types should be re-raised, not become an OpenEOApiException. - with pytest.raises(exc_class): - _ = abe_implementation.create_service( - user_id=TEST_USER, - process_graph=process_graph, - service_type="WMTS", - api_version=api_version, - configuration={} - ) + # These exception types should be re-raised, not become an OpenEOApiException. + with pytest.raises(exception_class): + _ = abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_version, + configuration={} + ) class TestInternalCollectionMetadata: diff --git a/tests/test_views.py b/tests/test_views.py index 8d078f01..0683e23e 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,4 +1,3 @@ -from datetime import datetime import itertools import logging import re @@ -12,7 +11,7 @@ from openeo_aggregator.config import AggregatorConfig from openeo_aggregator.testing import clock_mock from openeo_driver.errors import JobNotFoundException, JobNotFinishedException, \ - ProcessGraphInvalidException + ProcessGraphInvalidException, ProcessGraphMissingException from openeo_driver.backend import ServiceMetadata from openeo_driver.testing import ApiTester, TEST_USER_AUTH_HEADER, TEST_USER, TEST_USER_BEARER_TOKEN, DictSubSet, \ RegexMatcher @@ -1185,8 +1184,34 @@ def post_jobs(request: requests.Request, context): class TestSecondaryServices: + + @pytest.fixture + def service_metadata_wmts_foo(self): + return ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMTS service" + # not setting "created": This is used to test creating a service. + ) - # TODO: add view tests for list service types, list_services, servicfe_info + @pytest.fixture + def service_metadata_wms_bar(self): + return ServiceMetadata( + id="wms-bar", + process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, + url='https://oeo.net/wms/bar', + type="WMS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMS service" + # not setting "created": This is used to test creating a service. + ) def test_service_types_simple(self, api, backend1, backend2, requests_mock): """Given 2 backends but only 1 backend has a single service, then the aggregator @@ -1260,31 +1285,13 @@ def test_service_types_merging(self, api, backend1, backend2, requests_mock): expected_service_types.update(service_type_2) assert actual_service_types == expected_service_types - def test_service_info_api100(self, api100, backend1, backend2, requests_mock): + def test_service_info_api100( + self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar + ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" - service1 = ServiceMetadata( - id="wmts-foo", - process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, - url='https://oeo.net/wmts/foo', - type="WMTS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMTS service", - ) - service2 = ServiceMetadata( - id="wms-bar", - process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, - url='https://oeo.net/wms/bar', - type="WMS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={"layers": ["ndvi", "evi"]}, - title="Test WMS service", - ) - requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) + requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) api100.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) # Retrieve and verify the metadata for both services @@ -1299,7 +1306,7 @@ def test_service_info_api100(self, api100, backend1, backend2, requests_mock): attributes=resp.json["attributes"], title=resp.json["title"], ) - assert actual_service1 == service1 + assert actual_service1 == service_metadata_wmts_foo resp = api100.get("/services/wms-bar").assert_status_code(200) actual_service2 = ServiceMetadata( @@ -1312,53 +1319,48 @@ def test_service_info_api100(self, api100, backend1, backend2, requests_mock): attributes=resp.json["attributes"], title=resp.json["title"], ) - assert actual_service2 == service2 + assert actual_service2 == service_metadata_wms_bar - def test_service_info_api040(self, api040, backend1, backend2, requests_mock): + def test_service_info_api040( + self, api040, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar + ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" - service1 = ServiceMetadata( - id="wmts-foo", - process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, - url='https://oeo.net/wmts/foo', - type="WMTS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={"layers": ["ndvi", "evi"]}, - title="Test WMTS service", - ) - service2 = ServiceMetadata( - id="wms-bar", - process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, - url='https://oeo.net/wms/bar', - type="WMS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMS service", - ) - requests_mock.get(backend1 + "/services/wmts-foo", json=service1.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=service2.prepare_for_json()) + expected_service1 = service_metadata_wmts_foo + expected_service2 = service_metadata_wms_bar + requests_mock.get(backend1 + "/services/wmts-foo", json=expected_service1.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=expected_service2.prepare_for_json()) api040.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) # Retrieve and verify the metadata for both services resp = api040.get("/services/wmts-foo").assert_status_code(200) # required = ["id", "process", "url", "type", "enabled", "configuration", "attributes", "title"] - assert service1.id == resp.json["id"] - assert service1.process["process_graph"] == resp.json["process_graph"] - assert service1.url == resp.json["url"] - assert service1.type == resp.json["type"] - assert service1.title == resp.json["title"] - assert service1.attributes == resp.json["attributes"] + assert expected_service1.id == resp.json["id"] + assert expected_service1.process["process_graph"] == resp.json["process_graph"] + assert expected_service1.url == resp.json["url"] + assert expected_service1.type == resp.json["type"] + assert expected_service1.title == resp.json["title"] + assert expected_service1.attributes == resp.json["attributes"] resp = api040.get("/services/wms-bar").assert_status_code(200) - assert service2.id == resp.json["id"] - assert service2.process["process_graph"] == resp.json["process_graph"] - assert service2.url == resp.json["url"] - assert service2.type == resp.json["type"] - assert service2.title == resp.json["title"] - assert service2.attributes == resp.json["attributes"] - assert service2.attributes == resp.json["attributes"] + assert expected_service2.id == resp.json["id"] + assert expected_service2.process["process_graph"] == resp.json["process_graph"] + assert expected_service2.url == resp.json["url"] + assert expected_service2.type == resp.json["type"] + assert expected_service2.title == resp.json["title"] + assert expected_service2.attributes == resp.json["attributes"] + assert expected_service2.attributes == resp.json["attributes"] + + def test_service_info_wrong_id( + self, api, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar + ): + """When it gets a non-existent service ID, it returns HTTP Status 404, Not found.""" + + requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) + requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) + api.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) + + api.get("/services/doesnotexist").assert_status_code(404) def test_create_wmts_040(self, api040, requests_mock, backend1): api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) @@ -1422,6 +1424,30 @@ def test_create_wmts_100(self, api100, requests_mock, backend1): assert resp.headers['OpenEO-Identifier'] == 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' assert resp.headers['Location'] == expected_location + @pytest.mark.parametrize("exception_class", [ProcessGraphMissingException, ProcessGraphInvalidException]) + def test_create_wmts_100_reports_400_client_error(self, api100, requests_mock, backend1, exception_class): + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + # The process_graph/process format is slightly different between api v0.4 and v1.0 + post_data = { + "type": 'WMTS', + "process": { + "process_graph": process_graph, + "id": "filter_temporal_wmts" + }, + "custom_param": 45, + "title": "My Service", + "description": "Service description" + } + requests_mock.post( + backend1 + "/services", + exc=exception_class("Testing exception handling") + ) + + resp = api100.post('/services', json=post_data) + resp.status_code == 400 + class TestResilience: From 584d7dd0efc3f85b30eb9871252022439dd75780 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Tue, 8 Nov 2022 12:29:32 +0100 Subject: [PATCH 09/26] Issue #78 Adding more tests for secondary services --- tests/conftest.py | 3 +- tests/test_backend.py | 29 +++++++----- tests/test_views.py | 102 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 112 insertions(+), 22 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 04aaf2b8..b46a2e8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -159,10 +159,11 @@ def api(flask_app, api_version) -> ApiTester: """Get an ApiTester for each version. Useful when it easy to test several API versions with (mostly) the same test code. - But when the difference is too big, just write separate tests. + But when the difference is too big, just keep it simple and write separate tests. """ return get_api_version(flask_app, api_version) + @pytest.fixture def api040(flask_app: flask.Flask) -> ApiTester: return get_api040(flask_app) diff --git a/tests/test_backend.py b/tests/test_backend.py index 675d7043..5dcca768 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -360,32 +360,44 @@ def test_service_info_wrong_id( with pytest.raises(ServiceNotFoundException): _ = abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") - @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) - def test_create_service(self, multi_backend_connection, config, backend1, requests_mock, api_version): + def test_create_service(self, api, multi_backend_connection, config, backend1, requests_mock): """When it gets a correct params for a new service, it succesfully creates it.""" # Set up responses for creating the service in backend 1 expected_openeo_id = "wmts-foo" - expected_location = backend1 + "/services/wmts-foo" + location_backend_1 = backend1 + "/services/" + expected_openeo_id process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", headers={ "OpenEO-Identifier": expected_openeo_id, - "Location": expected_location + "Location": location_backend_1 }, status_code=201) - + service1 = ServiceMetadata( + id="wmts-foo", + process=process_graph, + url=location_backend_1, + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test WMTS service", + ) + requests_mock.get( + location_backend_1, + json=service1.prepare_for_json(), + status_code=200 + ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) actual_openeo_id = abe_implementation.create_service( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", - api_version=api_version, + api_version=api.api_version, configuration={} ) - assert actual_openeo_id == expected_openeo_id @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) @@ -393,7 +405,6 @@ def test_create_service(self, multi_backend_connection, config, backend1, reques def test_create_service_backend_raises_openeoapiexception( self, multi_backend_connection, config, backend1, requests_mock, api_version, exception_class ): - # for exc_class in [OpenEoApiError, OpenEoRestError]: # Set up responses for creating the service in backend 1: # This time the backend raises an error, one that will be reported as a OpenEOApiException. process_graph = {"foo": {"process_id": "foo", "arguments": {}}} @@ -401,7 +412,6 @@ def test_create_service_backend_raises_openeoapiexception( backend1 + "/services", exc=exception_class("Some server error"), ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) with pytest.raises(OpenEOApiException): @@ -427,7 +437,6 @@ def test_create_service_backend_reraises( backend1 + "/services", exc=exception_class("Some server error"), ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) # These exception types should be re-raised, not become an OpenEOApiException. diff --git a/tests/test_views.py b/tests/test_views.py index 0683e23e..54dcd20a 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -7,6 +7,7 @@ import requests from openeo.rest.connection import url_join +from openeo.rest import OpenEoApiError, OpenEoRestError from openeo_aggregator.backend import AggregatorCollectionCatalog from openeo_aggregator.config import AggregatorConfig from openeo_aggregator.testing import clock_mock @@ -1184,7 +1185,7 @@ def post_jobs(request: requests.Request, context): class TestSecondaryServices: - + @pytest.fixture def service_metadata_wmts_foo(self): return ServiceMetadata( @@ -1333,8 +1334,9 @@ def test_service_info_api040( api040.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) # Retrieve and verify the metadata for both services + # Here we compare attribute by attribute because in API version 0.4.0 the response + # has fewer properties, and some have a slightly different name and data structure. resp = api040.get("/services/wmts-foo").assert_status_code(200) - # required = ["id", "process", "url", "type", "enabled", "configuration", "attributes", "title"] assert expected_service1.id == resp.json["id"] assert expected_service1.process["process_graph"] == resp.json["process_graph"] assert expected_service1.url == resp.json["url"] @@ -1349,7 +1351,6 @@ def test_service_info_api040( assert expected_service2.type == resp.json["type"] assert expected_service2.title == resp.json["title"] assert expected_service2.attributes == resp.json["attributes"] - assert expected_service2.attributes == resp.json["attributes"] def test_service_info_wrong_id( self, api, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar @@ -1366,14 +1367,17 @@ def test_create_wmts_040(self, api040, requests_mock, backend1): api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) expected_openeo_id = 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' + # The aggregator MUST NOT point to the actual instance but to its own endpoint. + # This is handled by the openeo python driver in openeo_driver.views.services_post. expected_location = "/openeo/0.4.0/services/" + expected_openeo_id + # However, backend1 must report its OWN location. + location_backend_1 = backend1 + "/services" + expected_openeo_id process_graph = {"foo": {"process_id": "foo", "arguments": {}}} # The process_graph/process format is slightly different between api v0.4 and v1.0 post_data = { "type": 'WMTS', "process_graph": process_graph, - "custom_param": 45, "title": "My Service", "description": "Service description" } @@ -1382,7 +1386,7 @@ def test_create_wmts_040(self, api040, requests_mock, backend1): backend1 + "/services", headers={ "OpenEO-Identifier": expected_openeo_id, - "Location": expected_location + "Location": location_backend_1 }, status_code=201 ) @@ -1394,9 +1398,12 @@ def test_create_wmts_040(self, api040, requests_mock, backend1): def test_create_wmts_100(self, api100, requests_mock, backend1): api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - # used both to set up data and to validate at the end expected_openeo_id = 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' + # The aggregator MUST NOT point to the actual instance but to its own endpoint. + # This is handled by the openeo python driver in openeo_driver.views.services_post. expected_location = "/openeo/1.0.0/services/" + expected_openeo_id + # However, backend1 must report its OWN location. + location_backend_1 = backend1 + "/services" + expected_openeo_id process_graph = {"foo": {"process_id": "foo", "arguments": {}}} # The process_graph/process format is slightly different between api v0.4 and v1.0 @@ -1406,7 +1413,6 @@ def test_create_wmts_100(self, api100, requests_mock, backend1): "process_graph": process_graph, "id": "filter_temporal_wmts" }, - "custom_param": 45, "title": "My Service", "description": "Service description" } @@ -1414,7 +1420,7 @@ def test_create_wmts_100(self, api100, requests_mock, backend1): backend1 + "/services", headers={ "OpenEO-Identifier": expected_openeo_id, - "Location": expected_location + "Location": location_backend_1 }, status_code=201 ) @@ -1424,8 +1430,83 @@ def test_create_wmts_100(self, api100, requests_mock, backend1): assert resp.headers['OpenEO-Identifier'] == 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' assert resp.headers['Location'] == expected_location + # TODO: maybe testing specifically client error vs server error goes to far. It may be a bit too complicated. + # ProcessGraphMissingException and ProcessGraphInvalidException are well known reasons for a bad client request. + @pytest.mark.parametrize("exception_class", [ProcessGraphMissingException, ProcessGraphInvalidException]) + def test_create_wmts_reports_400_client_error_api040(self, api040, requests_mock, backend1, exception_class): + """When the backend raised an exception that we know represents incorrect input / client error, + then the aggregator's responds with an HTTP status code in the 400 range. + """ + api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + # The process_graph/process format is slightly different between api v0.4 and v1.0 + post_data = { + "type": 'WMTS', + "process_graph": process_graph, + "title": "My Service", + "description": "Service description" + } + # TODO: In theory we should make the backend report a HTTP 400 status and then the aggregator + # should also report HTTP 400. But in fact that comes back as HTTP 500. + requests_mock.post( + backend1 + "/services", + exc=exception_class("Testing exception handling") + ) + + resp = api040.post('/services', json=post_data) + assert resp.status_code == 400 + + # ProcessGraphMissingException and ProcessGraphInvalidException are well known reasons for a bad client request. @pytest.mark.parametrize("exception_class", [ProcessGraphMissingException, ProcessGraphInvalidException]) - def test_create_wmts_100_reports_400_client_error(self, api100, requests_mock, backend1, exception_class): + def test_create_wmts_reports_400_client_error_api100(self, api100, requests_mock, backend1, exception_class): + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + # The process_graph/process format is slightly different between api v0.4 and v1.0 + post_data = { + "type": 'WMTS', + "process": { + "process_graph": process_graph, + "id": "filter_temporal_wmts" + }, + "title": "My Service", + "description": "Service description" + } + # TODO: In theory we should make the backend report a HTTP 400 status and then the aggregator + # should also report HTTP 400. But in fact that comes back as HTTP 500. + requests_mock.post( + backend1 + "/services", + exc=exception_class("Testing exception handling") + ) + + resp = api100.post('/services', json=post_data) + assert resp.status_code == 400 + + # OpenEoApiError, OpenEoRestError: more general errors we can expect to lead to a HTTP 500 server error. + @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) + def test_create_wmts_reports_500_server_error_api040(self, api040, requests_mock, backend1, exception_class): + api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + + process_graph = {"foo": {"process_id": "foo", "arguments": {}}} + # The process_graph/process format is slightly different between api v0.4 and v1.0 + post_data = { + "type": 'WMTS', + "process_graph": process_graph, + "title": "My Service", + "description": "Service description" + } + requests_mock.post( + backend1 + "/services", + exc=exception_class("Testing exception handling") + ) + + resp = api040.post('/services', json=post_data) + assert resp.status_code == 500 + + # OpenEoApiError, OpenEoRestError: more general errors we can expect to lead to a HTTP 500 server error. + @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) + def test_create_wmts_reports_500_server_error_api100(self, api100, requests_mock, backend1, exception_class): api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) process_graph = {"foo": {"process_id": "foo", "arguments": {}}} @@ -1436,7 +1517,6 @@ def test_create_wmts_100_reports_400_client_error(self, api100, requests_mock, b "process_graph": process_graph, "id": "filter_temporal_wmts" }, - "custom_param": 45, "title": "My Service", "description": "Service description" } @@ -1446,7 +1526,7 @@ def test_create_wmts_100_reports_400_client_error(self, api100, requests_mock, b ) resp = api100.post('/services', json=post_data) - resp.status_code == 400 + assert resp.status_code == 500 class TestResilience: From 4097e0b8c8d8ea1f025878d40326fd1fbca6de41 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Tue, 8 Nov 2022 21:22:49 +0100 Subject: [PATCH 10/26] Issue #78 Implementing remove_service --- src/openeo_aggregator/backend.py | 36 ++++++++++- tests/test_backend.py | 105 +++++++++++++++++++++---------- 2 files changed, 107 insertions(+), 34 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index fdc1a5a0..5f7c8de3 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -827,7 +827,7 @@ def merge(services, to_add): try: services_json = con.get("/services").json() except Exception as e: - _log.warning("Failed to get services from {con.id}: {e!r}", exc_info=True) + _log.warning(f"Failed to get services from {con.id}: {e!r}", exc_info=True) continue if services_json: @@ -843,7 +843,7 @@ def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: try: service_json = con.get(f"/services/{service_id}").json() except Exception as e: - _log.debug("No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) + _log.debug(f"No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) continue else: return ServiceMetadata.from_dict(service_json) @@ -877,6 +877,34 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a return service.service_id + def remove_service(self, user_id: str, service_id: str) -> None: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" + # Search all services on the backends. + for con in self._backends: + try: + _ = con.get(f"/services/{service_id}") + except OpenEoApiError as e: + if e.http_status_code == 404: + _log.debug(f"No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) + continue + else: + _log.warning(f"Failed to get service {service_id!r} from {con.id}: {e!r}", exc_info=True) + raise e + except Exception as e: + _log.warning(f"Failed to get service {service_id!r} from {con.id}: {e!r}", exc_info=True) + raise e + + try: + response_del = con.delete(f"/services/{service_id}") + except Exception as e: + _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + raise OpenEOApiException( + f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}" + ) from e + + if response_del.status_code != 204: + raise OpenEOApiException(f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}") + class AggregatorBackendImplementation(OpenEoBackendImplementation): # No basic auth: OIDC auth is required (to get EGI Check-in eduperson_entitlement data) @@ -1076,3 +1104,7 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a configuration: dict) -> Tuple[str, str]: return self.secondary_services.create_service(user_id=user_id, process_graph=process_graph, service_type=service_type, api_version=api_version, configuration=configuration) + + def remove_service(self, user_id: str, service_id: str) -> None: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" + return self.secondary_services.remove_service(user_id=user_id, service_id=service_id) diff --git a/tests/test_backend.py b/tests/test_backend.py index 5dcca768..e213d11f 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -358,10 +358,10 @@ def test_service_info_wrong_id( ) with pytest.raises(ServiceNotFoundException): - _ = abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") + abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") def test_create_service(self, api, multi_backend_connection, config, backend1, requests_mock): - """When it gets a correct params for a new service, it succesfully creates it.""" + """When it gets a correct params for a new service, it successfully creates it.""" # Set up responses for creating the service in backend 1 expected_openeo_id = "wmts-foo" @@ -373,22 +373,9 @@ def test_create_service(self, api, multi_backend_connection, config, backend1, r "OpenEO-Identifier": expected_openeo_id, "Location": location_backend_1 }, - status_code=201) - service1 = ServiceMetadata( - id="wmts-foo", - process=process_graph, - url=location_backend_1, - type="WMTS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMTS service", - ) - requests_mock.get( - location_backend_1, - json=service1.prepare_for_json(), - status_code=200 + status_code=201 ) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) actual_openeo_id = abe_implementation.create_service( @@ -405,6 +392,8 @@ def test_create_service(self, api, multi_backend_connection, config, backend1, r def test_create_service_backend_raises_openeoapiexception( self, multi_backend_connection, config, backend1, requests_mock, api_version, exception_class ): + """When the backend raises a general exception the aggregator raises an OpenEOApiException.""" + # Set up responses for creating the service in backend 1: # This time the backend raises an error, one that will be reported as a OpenEOApiException. process_graph = {"foo": {"process_id": "foo", "arguments": {}}} @@ -415,13 +404,13 @@ def test_create_service_backend_raises_openeoapiexception( abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) with pytest.raises(OpenEOApiException): - _ = abe_implementation.create_service( - user_id=TEST_USER, - process_graph=process_graph, - service_type="WMTS", - api_version=api_version, - configuration={} - ) + abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_version, + configuration={} + ) @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) @pytest.mark.parametrize("exception_class", @@ -430,24 +419,76 @@ def test_create_service_backend_raises_openeoapiexception( def test_create_service_backend_reraises( self, multi_backend_connection, config, backend1, requests_mock, api_version, exception_class ): + """When the backend raises exception types that indicate client error / bad input data, + the aggregator raises and OpenEOApiException. + """ + # Set up responses for creating the service in backend 1 # This time the backend raises an error, one that will simply be re-raised/passed on as it is. process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", - exc=exception_class("Some server error"), + exc=exception_class("Some server error") ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) # These exception types should be re-raised, not become an OpenEOApiException. with pytest.raises(exception_class): - _ = abe_implementation.create_service( - user_id=TEST_USER, - process_graph=process_graph, - service_type="WMTS", - api_version=api_version, - configuration={} - ) + abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_version, + configuration={} + ) + + def test_remove_service(self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo): + """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) + abe_implementation = AggregatorBackendImplementation( + backends=multi_backend_connection, config=config + ) + + requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + requests_mock.get( + backend2 + "/services/wmts-foo", + status_code=404 + ) + requests_mock.delete(backend1 + "/services/wmts-foo", status_code=204) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + # Should not raise any exceptions. + abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + + @pytest.mark.parametrize("backend_status_code", [400, 500]) + def test_remove_service_backend_response_is_an_error_status( + self, multi_backend_connection, config, backend1, backend2, requests_mock, + service_metadata_wmts_foo, backend_status_code + ): + """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) + abe_implementation = AggregatorBackendImplementation( + backends=multi_backend_connection, config=config + ) + + requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + requests_mock.get( + backend2 + "/services/wmts-foo", + status_code=404 + ) + requests_mock.delete(backend1 + "/services/wmts-foo", status_code=backend_status_code) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + with pytest.raises(OpenEOApiException): + abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") class TestInternalCollectionMetadata: From 9a09b79c75fc7669f9f90838d5cbc50883abbb39 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Wed, 9 Nov 2022 17:15:42 +0100 Subject: [PATCH 11/26] Issue #78, correcting remove_service and improve tests --- src/openeo_aggregator/backend.py | 19 ++++++---- tests/test_backend.py | 61 +++++++++++++++++++++++--------- 2 files changed, 57 insertions(+), 23 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 5f7c8de3..397bbff7 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -880,11 +880,13 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a def remove_service(self, user_id: str, service_id: str) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" # Search all services on the backends. + service = None for con in self._backends: try: - _ = con.get(f"/services/{service_id}") + service = con.get(f"/services/{service_id}") except OpenEoApiError as e: if e.http_status_code == 404: + # Expected error _log.debug(f"No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) continue else: @@ -895,15 +897,20 @@ def remove_service(self, user_id: str, service_id: str) -> None: raise e try: - response_del = con.delete(f"/services/{service_id}") + con.delete(f"/services/{service_id}", expected_status=204) + except (OpenEoApiError, OpenEOApiException) as e: + # TODO: maybe we should just let these exception straight go to the caller without logging it here. + # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. + _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + raise except Exception as e: _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) raise OpenEOApiException( f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}" ) from e - if response_del.status_code != 204: - raise OpenEOApiException(f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}") + if not service: + raise ServiceNotFoundException(service_id) class AggregatorBackendImplementation(OpenEoBackendImplementation): @@ -1101,10 +1108,10 @@ def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: return self.secondary_services.service_info(user_id=user_id, service_id=service_id) def create_service(self, user_id: str, process_graph: dict, service_type: str, api_version: str, - configuration: dict) -> Tuple[str, str]: + configuration: dict) -> str: return self.secondary_services.create_service(user_id=user_id, process_graph=process_graph, service_type=service_type, api_version=api_version, configuration=configuration) def remove_service(self, user_id: str, service_id: str) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" - return self.secondary_services.remove_service(user_id=user_id, service_id=service_id) + self.secondary_services.remove_service(user_id=user_id, service_id=service_id) diff --git a/tests/test_backend.py b/tests/test_backend.py index e213d11f..dc088617 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -120,6 +120,9 @@ def test_file_formats_merging(self, multi_backend_connection, config, backend1, } } + +class TestAggregatorSecondaryServices: + def test_service_types_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): """Given 2 backends and only 1 backend has a single service type, then the aggregator returns that 1 service type's metadata. @@ -442,52 +445,76 @@ def test_create_service_backend_reraises( configuration={} ) - def test_remove_service(self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo): + def test_remove_service( + self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" - requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) - abe_implementation = AggregatorBackendImplementation( - backends=multi_backend_connection, config=config - ) - requests_mock.get( + # Also test that it can skip backends that don't have the service + m_get1 = requests_mock.get( backend1 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 + status_code=404 ) - requests_mock.get( + # Delete should succeed in backend2 so service should be present first. + m_get2 = requests_mock.get( backend2 + "/services/wmts-foo", - status_code=404 + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 ) - requests_mock.delete(backend1 + "/services/wmts-foo", status_code=204) + m_del = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - # Should not raise any exceptions. abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + # Make sure the aggregator asked the backend to remove the service. + assert m_del.called + + # Check the other mocks were called too, just to be sure. + assert m_get1.called + assert m_get2.called + @pytest.mark.parametrize("backend_status_code", [400, 500]) def test_remove_service_backend_response_is_an_error_status( self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, backend_status_code ): - """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" + # Will find it on the first backend, and it should skip the second backend so we don't add it to backend2. + requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + requests_mock.delete(backend1 + "/services/wmts-foo", status_code=backend_status_code) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + with pytest.raises(OpenEoApiError) as e: + abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + + # If the backend reports HTTP 400/500, we would expect the same status code from the aggregator. + assert e.value.http_status_code == backend_status_code + + def test_remove_service_service_id_not_found( + self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + ): + """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - + # Neither backend has the service available, and the aggregator should detect this. requests_mock.get( backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 + status_code=404 ) requests_mock.get( backend2 + "/services/wmts-foo", status_code=404 ) - requests_mock.delete(backend1 + "/services/wmts-foo", status_code=backend_status_code) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - with pytest.raises(OpenEOApiException): + with pytest.raises(ServiceNotFoundException): abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") From 7469979f477c360edb0ac8a41934612dabfc3cc1 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Wed, 9 Nov 2022 21:40:25 +0100 Subject: [PATCH 12/26] Issue #78 start implementing update_service and improve tests --- src/openeo_aggregator/backend.py | 69 ++++++++++++++++++++++++-------- tests/conftest.py | 21 +++++++--- tests/test_backend.py | 67 ++++++++++++++++++++++++++----- tests/test_views.py | 14 +++---- 4 files changed, 131 insertions(+), 40 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 397bbff7..4f3c0956 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -877,13 +877,13 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a return service.service_id - def remove_service(self, user_id: str, service_id: str) -> None: - """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" + def _find_connection_with_service_id(self, service_id: str) -> BackendConnection: + """Get connection for the backend that contains the service, return None if not found.""" + # Search all services on the backends. - service = None for con in self._backends: try: - service = con.get(f"/services/{service_id}") + _ = con.get(f"/services/{service_id}") except OpenEoApiError as e: if e.http_status_code == 404: # Expected error @@ -895,23 +895,52 @@ def remove_service(self, user_id: str, service_id: str) -> None: except Exception as e: _log.warning(f"Failed to get service {service_id!r} from {con.id}: {e!r}", exc_info=True) raise e + else: + return con - try: - con.delete(f"/services/{service_id}", expected_status=204) - except (OpenEoApiError, OpenEOApiException) as e: - # TODO: maybe we should just let these exception straight go to the caller without logging it here. - # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. - _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) - raise - except Exception as e: - _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) - raise OpenEOApiException( - f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}" - ) from e + return None + + def remove_service(self, user_id: str, service_id: str) -> None: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" + con = self._find_connection_with_service_id(service_id) + if not con: + raise ServiceNotFoundException(service_id) - if not service: + try: + con.delete(f"/services/{service_id}", expected_status=204) + except (OpenEoApiError, OpenEOApiException) as e: + # TODO: maybe we should just let these exception straight go to the caller without logging it here. + # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. + _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + raise + except Exception as e: + _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + raise OpenEOApiException( + f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}" + ) from e + + def update_service(self, user_id: str, service_id: str, process_graph: dict) -> None: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/update-service""" + + con = self._find_connection_with_service_id(service_id) + if not con: raise ServiceNotFoundException(service_id) + api_version = self._backends.api_version + try: + key = "process_graph" if api_version < ComparableVersion((1, 0, 0)) else "process" + con.patch(f"/services/{service_id}", json={key: process_graph}, expected_status=204) + except (OpenEoApiError, OpenEOApiException) as e: + # TODO: maybe we should just let these exception straight go to the caller without logging it here. + # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. + _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + raise + except Exception as e: + _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + raise OpenEOApiException( + f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}" + ) from e + class AggregatorBackendImplementation(OpenEoBackendImplementation): # No basic auth: OIDC auth is required (to get EGI Check-in eduperson_entitlement data) @@ -1115,3 +1144,9 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a def remove_service(self, user_id: str, service_id: str) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" self.secondary_services.remove_service(user_id=user_id, service_id=service_id) + + def update_service(self, user_id: str, service_id: str, process_graph: dict) -> None: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/update-service""" + self.secondary_services.update_service( + user_id=user_id, service_id=service_id, process_graph=process_graph + ) \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index b46a2e8d..2cd61654 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture -def backend1(requests_mock) -> str: +def backend1(requests_mock,) -> str: domain = "https://b1.test/v1" # TODO: how to work with different API versions? requests_mock.get(domain + "/", json={"api_version": "1.0.0"}) @@ -33,6 +33,17 @@ def backend2(requests_mock) -> str: return domain +# as "lib_requests_mock": make a distinction with the pytest fixture that has the same name +import requests_mock as lib_requests_mock + +def set_backend_to_api_version(requests_mock: lib_requests_mock.Mocker, domain: str, api_version: str) -> str: + """Helper function to make the backend connection use the expected API version.""" + + # TODO: would like a nicer solution to make the backend fixtures match the expected API version. + # Good enough for now tough, just have to remember to call it in your test. + return requests_mock.get(f"{domain}/", json={"api_version": api_version}) + + @pytest.fixture def main_test_oidc_issuer() -> str: """ @@ -137,12 +148,12 @@ def backend_implementation(flask_app) -> AggregatorBackendImplementation: @pytest.fixture(params=["0.4.0", "1.0.0"]) -def api_version(request): +def api_version_fixture(request): """To go through all relevant API versions""" return request.param -def get_api_version(flask_app, api_version) -> ApiTester: +def get_api_tester(flask_app, api_version) -> ApiTester: return ApiTester(api_version=api_version, client=flask_app.test_client()) @@ -155,13 +166,13 @@ def get_api100(flask_app: flask.Flask) -> ApiTester: @pytest.fixture -def api(flask_app, api_version) -> ApiTester: +def api_tester(flask_app, api_version_fixture) -> ApiTester: """Get an ApiTester for each version. Useful when it easy to test several API versions with (mostly) the same test code. But when the difference is too big, just keep it simple and write separate tests. """ - return get_api_version(flask_app, api_version) + return get_api_tester(flask_app, api_version_fixture) @pytest.fixture diff --git a/tests/test_backend.py b/tests/test_backend.py index dc088617..fff3c03f 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -14,8 +14,8 @@ from openeo_driver.users.oidc import OidcProvider from openeo_driver.errors import ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException from openeo.rest import OpenEoApiError, OpenEoRestError -from .conftest import DEFAULT_MEMOIZER_CONFIG - +from .conftest import DEFAULT_MEMOIZER_CONFIG, set_backend_to_api_version + TEST_USER = "Mr.Test" @@ -363,10 +363,11 @@ def test_service_info_wrong_id( with pytest.raises(ServiceNotFoundException): abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") - def test_create_service(self, api, multi_backend_connection, config, backend1, requests_mock): + def test_create_service(self, api_tester, multi_backend_connection, config, backend1, requests_mock): """When it gets a correct params for a new service, it successfully creates it.""" # Set up responses for creating the service in backend 1 + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) expected_openeo_id = "wmts-foo" location_backend_1 = backend1 + "/services/" + expected_openeo_id process_graph = {"foo": {"process_id": "foo", "arguments": {}}} @@ -385,20 +386,21 @@ def test_create_service(self, api, multi_backend_connection, config, backend1, r user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", - api_version=api.api_version, + api_version=api_tester.api_version, configuration={} ) assert actual_openeo_id == expected_openeo_id - @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) def test_create_service_backend_raises_openeoapiexception( - self, multi_backend_connection, config, backend1, requests_mock, api_version, exception_class + self, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, exception_class ): """When the backend raises a general exception the aggregator raises an OpenEOApiException.""" # Set up responses for creating the service in backend 1: # This time the backend raises an error, one that will be reported as a OpenEOApiException. + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", @@ -411,16 +413,15 @@ def test_create_service_backend_raises_openeoapiexception( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", - api_version=api_version, + api_version=api_tester.api_version, configuration={} ) - @pytest.mark.parametrize("api_version", ["0.4.0", "1.0.0", "1.1.0"]) @pytest.mark.parametrize("exception_class", [ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException] ) def test_create_service_backend_reraises( - self, multi_backend_connection, config, backend1, requests_mock, api_version, exception_class + self, api_tester, multi_backend_connection, config, backend1, requests_mock, exception_class ): """When the backend raises exception types that indicate client error / bad input data, the aggregator raises and OpenEOApiException. @@ -428,6 +429,7 @@ def test_create_service_backend_reraises( # Set up responses for creating the service in backend 1 # This time the backend raises an error, one that will simply be re-raised/passed on as it is. + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", @@ -441,7 +443,7 @@ def test_create_service_backend_reraises( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", - api_version=api_version, + api_version=api_tester.api_version, configuration={} ) @@ -475,7 +477,7 @@ def test_remove_service( @pytest.mark.parametrize("backend_status_code", [400, 500]) def test_remove_service_backend_response_is_an_error_status( - self, multi_backend_connection, config, backend1, backend2, requests_mock, + self, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo, backend_status_code ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" @@ -517,6 +519,49 @@ def test_remove_service_service_id_not_found( with pytest.raises(ServiceNotFoundException): abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + def test_update_service( + self, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + ): + """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + + # Also test that it can skip backends that don't have the service + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + mock_get1 = requests_mock.get( + backend1 + "/services/wmts-foo", + status_code=404 + ) + # Update should succeed in backend2 so service should be present first. + service_metadata_wmts_foo + mock_get2 = requests_mock.get( + backend2 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + mock_patch = requests_mock.patch( + backend2 + "/services/wmts-foo", + status_code=204, + ) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} + + abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + + # # Make sure the aggregator asked the backend to remove the service. + assert mock_patch.called + from openeo.capabilities import ComparableVersion + comp_api_version = ComparableVersion(api_tester.api_version) + if comp_api_version < ComparableVersion((1, 0, 0)): + expected_process = {"process_graph": process_graph_after} + else: + expected_process = {"process": process_graph_after} + + assert mock_patch.last_request.json() == expected_process + + # Check the other mocks were called too, just to be sure. + assert mock_get1.called + assert mock_get2.called + class TestInternalCollectionMetadata: diff --git a/tests/test_views.py b/tests/test_views.py index 54dcd20a..85f38c18 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1214,7 +1214,7 @@ def service_metadata_wms_bar(self): # not setting "created": This is used to test creating a service. ) - def test_service_types_simple(self, api, backend1, backend2, requests_mock): + def test_service_types_simple(self, api_tester, backend1, backend2, requests_mock): """Given 2 backends but only 1 backend has a single service, then the aggregator returns that 1 service's metadata. """ @@ -1242,10 +1242,10 @@ def test_service_types_simple(self, api, backend1, backend2, requests_mock): requests_mock.get(backend1 + "/service_types", json=single_service_type) requests_mock.get(backend2 + "/service_types", json=single_service_type) - resp = api.get('/service_types').assert_status_code(200) + resp = api_tester.get('/service_types').assert_status_code(200) assert resp.json == single_service_type - def test_service_types_merging(self, api, backend1, backend2, requests_mock): + def test_service_types_merging(self, api_tester, backend1, backend2, requests_mock): """Given 2 backends with each 1 service, then the aggregator lists both services.""" service_type_1 = { "WMTS": { @@ -1279,7 +1279,7 @@ def test_service_types_merging(self, api, backend1, backend2, requests_mock): requests_mock.get(backend1 + "/service_types", json=service_type_1) requests_mock.get(backend2 + "/service_types", json=service_type_2) - resp = api.get("/service_types").assert_status_code(200) + resp = api_tester.get("/service_types").assert_status_code(200) actual_service_types = resp.json expected_service_types = dict(service_type_1) @@ -1353,15 +1353,15 @@ def test_service_info_api040( assert expected_service2.attributes == resp.json["attributes"] def test_service_info_wrong_id( - self, api, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar + self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a non-existent service ID, it returns HTTP Status 404, Not found.""" requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) - api.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) + api_tester.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) - api.get("/services/doesnotexist").assert_status_code(404) + api_tester.get("/services/doesnotexist").assert_status_code(404) def test_create_wmts_040(self, api040, requests_mock, backend1): api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) From d68a32c827e1946250c6874ab18e6297e8bb5105 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Wed, 9 Nov 2022 22:37:21 +0100 Subject: [PATCH 13/26] Issue #78 add more tests in test_views.py --- src/openeo_aggregator/backend.py | 7 +++ tests/test_backend.py | 29 +++++----- tests/test_views.py | 96 +++++++++++++++++++++++++++++++- 3 files changed, 117 insertions(+), 15 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 4f3c0956..aa1599bc 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -812,6 +812,7 @@ def merge(formats: dict, to_add: dict): def list_services(self, user_id: str) -> List[ServiceMetadata]: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-services""" + # TODO: user_id is not used, how to authenticate when we use the BackendConnection? all_services = [] def merge(services, to_add): @@ -837,6 +838,7 @@ def merge(services, to_add): def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-service""" + # TODO: user_id is not used, how to authenticate when we use the BackendConnection? # TODO: can there ever be a service with the same ID in multiple back-ends? (For the same user) for con in self._backends: @@ -855,6 +857,8 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a """ https://openeo.org/documentation/1.0/developers/api/reference.html#operation/create-service """ + # TODO: user_id is not used, how to authenticate when we use the BackendConnection? + # TODO: configuration is not used. What to do with it? backend_id = self._processing.get_backend_for_process_graph( process_graph=process_graph, api_version=api_version @@ -902,6 +906,8 @@ def _find_connection_with_service_id(self, service_id: str) -> BackendConnection def remove_service(self, user_id: str, service_id: str) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" + # TODO: user_id is not used, how to authenticate when we use the BackendConnection? + con = self._find_connection_with_service_id(service_id) if not con: raise ServiceNotFoundException(service_id) @@ -921,6 +927,7 @@ def remove_service(self, user_id: str, service_id: str) -> None: def update_service(self, user_id: str, service_id: str, process_graph: dict) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/update-service""" + # TODO: user_id is not used, how to authenticate when we use the BackendConnection? con = self._find_connection_with_service_id(service_id) if not con: diff --git a/tests/test_backend.py b/tests/test_backend.py index fff3c03f..e4ccc1a8 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -447,40 +447,43 @@ def test_create_service_backend_reraises( configuration={} ) - def test_remove_service( + def test_remove_service_succeeds( self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): - """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" # Also test that it can skip backends that don't have the service - m_get1 = requests_mock.get( + mock_get1 = requests_mock.get( backend1 + "/services/wmts-foo", status_code=404 ) # Delete should succeed in backend2 so service should be present first. - m_get2 = requests_mock.get( + mock_get2 = requests_mock.get( backend2 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json(), status_code=200 ) - m_del = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) + mock_delete = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") # Make sure the aggregator asked the backend to remove the service. - assert m_del.called + assert mock_delete.called # Check the other mocks were called too, just to be sure. - assert m_get1.called - assert m_get2.called + assert mock_get1.called + assert mock_get2.called + # TODO: it fails for the test case where the backend reports HTTP 400, because along the way the aggregator turns it into a HTTP 500. + # Also, I'm not sure is this test is the way to go. @pytest.mark.parametrize("backend_status_code", [400, 500]) def test_remove_service_backend_response_is_an_error_status( self, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo, backend_status_code ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" + # Will find it on the first backend, and it should skip the second backend so we don't add it to backend2. requests_mock.get( backend1 + "/services/wmts-foo", @@ -494,16 +497,14 @@ def test_remove_service_backend_response_is_an_error_status( abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") # If the backend reports HTTP 400/500, we would expect the same status code from the aggregator. + # TODO: Statement above is an assumption. Is that really what we expect? assert e.value.http_status_code == backend_status_code def test_remove_service_service_id_not_found( self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" - requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) - abe_implementation = AggregatorBackendImplementation( - backends=multi_backend_connection, config=config - ) + # Neither backend has the service available, and the aggregator should detect this. requests_mock.get( backend1 + "/services/wmts-foo", @@ -519,10 +520,10 @@ def test_remove_service_service_id_not_found( with pytest.raises(ServiceNotFoundException): abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") - def test_update_service( + def test_update_service_succeeds( self, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): - """When it gets a correct service ID, it returns the expected ServiceMetadata.""" + """When it receives an existing service ID and a correct payload, it updates the expected service.""" # Also test that it can skip backends that don't have the service set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) diff --git a/tests/test_views.py b/tests/test_views.py index 85f38c18..895fd39a 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -16,7 +16,7 @@ from openeo_driver.backend import ServiceMetadata from openeo_driver.testing import ApiTester, TEST_USER_AUTH_HEADER, TEST_USER, TEST_USER_BEARER_TOKEN, DictSubSet, \ RegexMatcher -from .conftest import assert_dict_subset, get_api100, get_flask_app +from .conftest import assert_dict_subset, get_api100, get_flask_app, set_backend_to_api_version class TestGeneral: @@ -1528,6 +1528,100 @@ def test_create_wmts_reports_500_server_error_api100(self, api100, requests_mock resp = api100.post('/services', json=post_data) assert resp.status_code == 500 + def test_remove_service_succeeds( + self, api_tester, requests_mock, backend1, backend2, service_metadata_wmts_foo + ): + """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + + # Also test that it can skip backends that don't have the service + requests_mock.get( + backend1 + "/services/wmts-foo", + status_code=404 + ) + # Delete should succeed in backend2 so service should be present first. + requests_mock.get( + backend2 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + mock_delete = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) + + resp = api_tester.delete("/services/wmts-foo") + + assert resp.status_code == 204 + # Make sure the aggregator asked the backend to remove the service. + assert mock_delete.called + + # TODO: it fails for the test case where the backend reports HTTP 400, because along the way the aggregator turns it into a HTTP 500. + # Also, I'm not sure is this test is the way to go. + @pytest.mark.parametrize("backend_status_code", [400, 500]) + def test_remove_service_backend_response_is_an_error_status( + self, api_tester, requests_mock, backend1, backend2, + service_metadata_wmts_foo, backend_status_code + ): + """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + + # Will find it on the first backend, and it should skip the second backend so we don't add it to backend2. + requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + mock_delete = requests_mock.delete( + backend1 + "/services/wmts-foo", + status_code=backend_status_code, + json={ + "id": "936DA01F-9ABD-4D9D-80C7-02AF85C822A8", + "code": "ErrorRemovingService", + "message": "Service 'wmts-foo' could not be removed.", + "url": "https://example.openeo.org/docs/errors/SampleError" + } + ) + + resp = api_tester.delete("/services/wmts-foo") + + assert resp.status_code == backend_status_code + # Make sure the aggregator asked the backend to remove the service. + assert mock_delete.called + + def test_remove_service_service_id_not_found( + self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo + ): + """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + + # Neither backend has the service available, and the aggregator should detect this. + mock_get1 = requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=404 + ) + mock_get2 = requests_mock.get( + backend2 + "/services/wmts-foo", + status_code=404, + # json={ + # "id": "936DA01F-9ABD-4D9D-80C7-02AF85C822A8", + # "code": "ServiceNotFound", + # "message": "Service 'wmts-foo' does not exist.", + # "url": "https://example.openeo.org/docs/errors/SampleError" + # } + ) + + resp = api_tester.delete("/services/wmts-foo") + + assert resp.status_code == 404 + # Make sure the aggregator asked the backend to remove the service. + assert mock_get1.called + assert mock_get2.called + class TestResilience: From a66549393bf0fa362438f822ee9b46ecd59e23b0 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Thu, 10 Nov 2022 09:47:03 +0100 Subject: [PATCH 14/26] Issue #78 Removed test case for remove_service that is not so useful. --- tests/test_backend.py | 10 +++------- tests/test_views.py | 12 ++++-------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index e4ccc1a8..3f03a41d 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -475,12 +475,8 @@ def test_remove_service_succeeds( assert mock_get1.called assert mock_get2.called - # TODO: it fails for the test case where the backend reports HTTP 400, because along the way the aggregator turns it into a HTTP 500. - # Also, I'm not sure is this test is the way to go. - @pytest.mark.parametrize("backend_status_code", [400, 500]) def test_remove_service_backend_response_is_an_error_status( - self, multi_backend_connection, config, backend1, requests_mock, - service_metadata_wmts_foo, backend_status_code + self, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" @@ -490,7 +486,7 @@ def test_remove_service_backend_response_is_an_error_status( json=service_metadata_wmts_foo.prepare_for_json(), status_code=200 ) - requests_mock.delete(backend1 + "/services/wmts-foo", status_code=backend_status_code) + requests_mock.delete(backend1 + "/services/wmts-foo", status_code=500) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) with pytest.raises(OpenEoApiError) as e: @@ -498,7 +494,7 @@ def test_remove_service_backend_response_is_an_error_status( # If the backend reports HTTP 400/500, we would expect the same status code from the aggregator. # TODO: Statement above is an assumption. Is that really what we expect? - assert e.value.http_status_code == backend_status_code + assert e.value.http_status_code == 500 def test_remove_service_service_id_not_found( self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo diff --git a/tests/test_views.py b/tests/test_views.py index 895fd39a..ec616168 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1555,12 +1555,8 @@ def test_remove_service_succeeds( # Make sure the aggregator asked the backend to remove the service. assert mock_delete.called - # TODO: it fails for the test case where the backend reports HTTP 400, because along the way the aggregator turns it into a HTTP 500. - # Also, I'm not sure is this test is the way to go. - @pytest.mark.parametrize("backend_status_code", [400, 500]) def test_remove_service_backend_response_is_an_error_status( - self, api_tester, requests_mock, backend1, backend2, - service_metadata_wmts_foo, backend_status_code + self, api_tester, requests_mock, backend1, backend2, service_metadata_wmts_foo ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) @@ -1575,7 +1571,7 @@ def test_remove_service_backend_response_is_an_error_status( ) mock_delete = requests_mock.delete( backend1 + "/services/wmts-foo", - status_code=backend_status_code, + status_code=500, json={ "id": "936DA01F-9ABD-4D9D-80C7-02AF85C822A8", "code": "ErrorRemovingService", @@ -1586,8 +1582,8 @@ def test_remove_service_backend_response_is_an_error_status( resp = api_tester.delete("/services/wmts-foo") - assert resp.status_code == backend_status_code - # Make sure the aggregator asked the backend to remove the service. + assert resp.status_code == 500 + # Make sure the aggregator effectively asked the backend to remove the service. assert mock_delete.called def test_remove_service_service_id_not_found( From 35e554c55acfb359c8f9413f6f83c19e1cb1830a Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Thu, 10 Nov 2022 15:05:22 +0100 Subject: [PATCH 15/26] Issue #78 WIP on tests for Secondary Services --- src/openeo_aggregator/backend.py | 43 ++++---- tests/test_backend.py | 177 ++++++++++++++++++++++--------- tests/test_views.py | 110 +++++++++++++++++-- 3 files changed, 248 insertions(+), 82 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index aa1599bc..70a92356 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -824,15 +824,17 @@ def merge(services, to_add): # Collect all services from the backends. for con in self._backends: - services_json = None - try: - services_json = con.get("/services").json() - except Exception as e: - _log.warning(f"Failed to get services from {con.id}: {e!r}", exc_info=True) - continue + # with con.authenticated_from_request(flask.request): + with con.authenticated_from_request(request=flask.request, user=User(user_id)): + services_json = None + try: + services_json = con.get("/services").json() + except Exception as e: + _log.warning(f"Failed to get services from {con.id}: {e!r}", exc_info=True) + continue - if services_json: - merge(all_services, services_json) + if services_json: + merge(all_services, services_json) return all_services @@ -891,13 +893,13 @@ def _find_connection_with_service_id(self, service_id: str) -> BackendConnection except OpenEoApiError as e: if e.http_status_code == 404: # Expected error - _log.debug(f"No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) + _log.debug(f"No service with ID={service_id!r} in backend with ID={con.id!r}: {e!r}", exc_info=True) continue else: - _log.warning(f"Failed to get service {service_id!r} from {con.id}: {e!r}", exc_info=True) + _log.warning(f"Failed to get service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise e except Exception as e: - _log.warning(f"Failed to get service {service_id!r} from {con.id}: {e!r}", exc_info=True) + _log.warning(f"Failed to get service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise e else: return con @@ -917,12 +919,12 @@ def remove_service(self, user_id: str, service_id: str) -> None: except (OpenEoApiError, OpenEOApiException) as e: # TODO: maybe we should just let these exception straight go to the caller without logging it here. # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. - _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + _log.warning(f"Failed to delete service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise except Exception as e: - _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + _log.warning(f"Failed to delete service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise OpenEOApiException( - f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}" + f"Failed to delete service {service_id!r} on backend {con.id!r}: {e!r}" ) from e def update_service(self, user_id: str, service_id: str, process_graph: dict) -> None: @@ -935,17 +937,20 @@ def update_service(self, user_id: str, service_id: str, process_graph: dict) -> api_version = self._backends.api_version try: - key = "process_graph" if api_version < ComparableVersion((1, 0, 0)) else "process" - con.patch(f"/services/{service_id}", json={key: process_graph}, expected_status=204) + if api_version < ComparableVersion((1, 0, 0)): + json = {"process_graph": process_graph} + else: + json = {"process": {"process_graph": process_graph}} + con.patch(f"/services/{service_id}", json=json, expected_status=204) except (OpenEoApiError, OpenEOApiException) as e: # TODO: maybe we should just let these exception straight go to the caller without logging it here. # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. - _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + _log.warning(f"Failed to update service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise except Exception as e: - _log.warning(f"Failed to delete service {service_id!r} from {con.id}: {e!r}", exc_info=True) + _log.warning(f"Failed to update service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise OpenEOApiException( - f"Failed to delete secondary service with id {service_id!r} on backend {con.id!r}: {e!r}" + f"Failed to update service {service_id!r} from {con.id!r}: {e!r}" ) from e diff --git a/tests/test_backend.py b/tests/test_backend.py index 3f03a41d..a9358779 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -12,13 +12,18 @@ ServiceNotFoundException from openeo_driver.testing import DictSubSet from openeo_driver.users.oidc import OidcProvider +from openeo_driver.users.auth import HttpAuthHandler from openeo_driver.errors import ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException +from openeo.capabilities import ComparableVersion from openeo.rest import OpenEoApiError, OpenEoRestError from .conftest import DEFAULT_MEMOIZER_CONFIG, set_backend_to_api_version - -TEST_USER = "Mr.Test" +TEST_USER = "Mr.Test" +TEST_USER_BEARER_TOKEN = "basic//" + HttpAuthHandler.build_basic_access_token(user_id=TEST_USER) +TEST_USER_AUTH_HEADER = { + "Authorization": "Bearer " + TEST_USER_BEARER_TOKEN +} class TestAggregatorBackendImplementation: @@ -228,49 +233,60 @@ def service_metadata_wms_bar(self): ) def test_list_services_simple( - self, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """Given 2 backends but only 1 backend has a single service, then the aggregator returns that 1 service's metadata. """ + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER + services1 = {"services": [service_metadata_wmts_foo.prepare_for_json()], "links": []} services2 = {} - requests_mock.get(backend1 + "/services", json=services1) - requests_mock.get(backend2 + "/services", json=services2) + requests_mock.get(backend1 + "/services", json=services1, headers=headers) + requests_mock.get(backend2 + "/services", json=services2, headers=headers) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - actual_services = abe_implementation.list_services(user_id=TEST_USER) + with flask_app.test_request_context(headers=headers): + actual_services = abe_implementation.list_services(user_id=TEST_USER) - # Construct expected result. We have get just data from the service in services1 - # (there is only one) for conversion to a ServiceMetadata. - the_service = services1["services"][0] - expected_services = [ - ServiceMetadata.from_dict(the_service) - ] - assert actual_services == expected_services + # Construct expected result. We have get just data from the service in services1 + # (there is only one) for conversion to a ServiceMetadata. + the_service = services1["services"][0] + expected_services = [ + ServiceMetadata.from_dict(the_service) + ] + assert actual_services == expected_services def test_list_services_merged( - self, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """Given 2 backends with each 1 service, then the aggregator lists both services.""" + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER + services1 = {"services": [service_metadata_wmts_foo.prepare_for_json()], "links": []} services2 = {"services": [service_metadata_wms_bar.prepare_for_json()], "links": []} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - actual_services = abe_implementation.list_services(user_id=TEST_USER) + with flask_app.test_request_context(headers=headers): + actual_services = abe_implementation.list_services(user_id=TEST_USER) - expected_services = [service_metadata_wmts_foo, service_metadata_wms_bar] - assert sorted(actual_services) == sorted(expected_services) + expected_services = [service_metadata_wmts_foo, service_metadata_wms_bar] + assert sorted(actual_services) == sorted(expected_services) def test_list_services_merged_multiple( - self, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """Given multiple services across 2 backends, the aggregator lists all service types from all backends.""" + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER + services1 = { "services": [{ "id": "wms-nvdi", @@ -318,20 +334,21 @@ def test_list_services_merged_multiple( backends=multi_backend_connection, config=config ) - actual_services = abe_implementation.list_services(user_id=TEST_USER) + with flask_app.test_request_context(headers=headers): + actual_services = abe_implementation.list_services(user_id=TEST_USER) - # Construct expected result. We have get just data from the service in - # services1 (there is only one) for conversion to a ServiceMetadata. - # TODO: do we need to take care of the links part in the JSON as well? - service1 = services1["services"][0] - service1_md = ServiceMetadata.from_dict(service1) - expected_services = [ - service1_md, service_metadata_wmts_foo, service_metadata_wms_bar - ] + # Construct expected result. We have get just data from the service in + # services1 (there is only one) for conversion to a ServiceMetadata. + # TODO: do we need to take care of the links part in the JSON as well? + service1 = services1["services"][0] + service1_md = ServiceMetadata.from_dict(service1) + expected_services = [ + service1_md, service_metadata_wmts_foo, service_metadata_wms_bar + ] - assert sorted(actual_services) == sorted(expected_services) + assert sorted(actual_services) == sorted(expected_services) - def test_service_info( + def test_service_info_succeeds( self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): @@ -363,7 +380,7 @@ def test_service_info_wrong_id( with pytest.raises(ServiceNotFoundException): abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") - def test_create_service(self, api_tester, multi_backend_connection, config, backend1, requests_mock): + def test_create_service_succeeds(self, api_tester, multi_backend_connection, config, backend1, requests_mock): """When it gets a correct params for a new service, it successfully creates it.""" # Set up responses for creating the service in backend 1 @@ -475,6 +492,42 @@ def test_remove_service_succeeds( assert mock_get1.called assert mock_get2.called + def test_remove_service_service_id_not_found( + self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + ): + """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" + + # Neither backend has the service available, and the aggregator should detect this. + mock_get1 = requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=404 + ) + mock_get2 = requests_mock.get( + backend2 + "/services/wmts-foo", + status_code=404 + ) + + # These requests should not be executed, so check they are not called. + mock_delete1 = requests_mock.delete( + backend2 + "/services/wmts-foo", + status_code=204 + ) + mock_delete2 = requests_mock.delete( + backend2 + "/services/wmts-foo", + status_code=204 + ) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + with pytest.raises(ServiceNotFoundException): + abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + + assert not mock_delete1.called + assert not mock_delete2.called + # Check the other mocks were called too, just to be sure. + assert mock_get1.called + assert mock_get2.called + def test_remove_service_backend_response_is_an_error_status( self, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo ): @@ -496,26 +549,7 @@ def test_remove_service_backend_response_is_an_error_status( # TODO: Statement above is an assumption. Is that really what we expect? assert e.value.http_status_code == 500 - def test_remove_service_service_id_not_found( - self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo - ): - """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" - - # Neither backend has the service available, and the aggregator should detect this. - requests_mock.get( - backend1 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=404 - ) - requests_mock.get( - backend2 + "/services/wmts-foo", - status_code=404 - ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - - with pytest.raises(ServiceNotFoundException): - abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") - + # TODO: this test still fails with API version 1.0.0 def test_update_service_succeeds( self, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): @@ -528,8 +562,7 @@ def test_update_service_succeeds( backend1 + "/services/wmts-foo", status_code=404 ) - # Update should succeed in backend2 so service should be present first. - service_metadata_wmts_foo + # Update should succeed in backend2 so service should be present mock_get2 = requests_mock.get( backend2 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json(), @@ -546,7 +579,7 @@ def test_update_service_succeeds( # # Make sure the aggregator asked the backend to remove the service. assert mock_patch.called - from openeo.capabilities import ComparableVersion + # TODO: I am not too sure this json payload is correct. Check with codebases of other backend drivers. comp_api_version = ComparableVersion(api_tester.api_version) if comp_api_version < ComparableVersion((1, 0, 0)): expected_process = {"process_graph": process_graph_after} @@ -559,6 +592,44 @@ def test_update_service_succeeds( assert mock_get1.called assert mock_get2.called + def test_update_service_service_id_not_found( + self, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + ): + """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" + + # Also test that it can skip backends that don't have the service + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + mock_get1 = requests_mock.get( + backend1 + "/services/wmts-foo", + status_code=404 + ) + mock_get2 = requests_mock.get( + backend2 + "/services/wmts-foo", + status_code=404 + ) + + # These requests should not be executed, so check they are not called. + mock_patch1 = requests_mock.patch( + backend1 + "/services/wmts-foo", + status_code=204, + ) + mock_patch2 = requests_mock.patch( + backend2 + "/services/wmts-foo", + status_code=204, + ) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} + + with pytest.raises(ServiceNotFoundException): + abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + + assert not mock_patch1.called + assert not mock_patch2.called + # Check the other mocks were called too, just to be sure. + assert mock_get1.called + assert mock_get2.called + class TestInternalCollectionMetadata: diff --git a/tests/test_views.py b/tests/test_views.py index ec616168..d9d77ee1 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -6,6 +6,7 @@ import pytest import requests +from openeo.capabilities import ComparableVersion from openeo.rest.connection import url_join from openeo.rest import OpenEoApiError, OpenEoRestError from openeo_aggregator.backend import AggregatorCollectionCatalog @@ -1555,6 +1556,38 @@ def test_remove_service_succeeds( # Make sure the aggregator asked the backend to remove the service. assert mock_delete.called + def test_remove_service_service_id_not_found( + self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo + ): + """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + + # Neither backend has the service available, and the aggregator should detect this. + mock_get1 = requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=404 + ) + mock_get2 = requests_mock.get( + backend2 + "/services/wmts-foo", + status_code=404, + ) + mock_delete = requests_mock.delete( + backend2 + "/services/wmts-foo", + status_code=204, # deliberately avoid 404 so we know 404 comes from aggregator. + ) + + resp = api_tester.delete("/services/wmts-foo") + + assert resp.status_code == 404 + # Verify the aggregator did not call the backend to remove the service. + assert not mock_delete.called + # Verify the aggregator did query the backends to find the service. + assert mock_get1.called + assert mock_get2.called + def test_remove_service_backend_response_is_an_error_status( self, api_tester, requests_mock, backend1, backend2, service_metadata_wmts_foo ): @@ -1583,10 +1616,53 @@ def test_remove_service_backend_response_is_an_error_status( resp = api_tester.delete("/services/wmts-foo") assert resp.status_code == 500 - # Make sure the aggregator effectively asked the backend to remove the service. + # Verify the aggregator effectively asked the backend to remove the service, + # so we can reasonably assume that is where the error came from. assert mock_delete.called - def test_remove_service_service_id_not_found( + def test_update_service_service_succeeds( + self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo + ): + """When it receives an existing service ID and a correct payload, it updates the expected service.""" + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + + # Also test that it can skip backends that don't have the service + mock_get1 = requests_mock.get( + backend1 + "/services/wmts-foo", + status_code=404 + ) + mock_get2 = requests_mock.get( + backend2 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + mock_patch = requests_mock.patch( + backend2 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=204 + ) + + comp_version = ComparableVersion(api_tester.api_version) + process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} + if comp_version < ComparableVersion((1, 0, 0)): + json_payload = {"process_graph": process_graph} + else: + json_payload = {"process": {"process_graph": process_graph}} + + resp = api_tester.patch("/services/wmts-foo", json=json_payload) + + assert resp.status_code == 204 + # Make sure the aggregator asked the backend to update the service. + assert mock_patch.called + assert mock_patch.last_request.json() == json_payload + + # Check other mocks were called, to be sure it searched before updating. + assert mock_get1.called + assert mock_get2.called + + def test_update_service_service_id_not_found( self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" @@ -1603,18 +1679,32 @@ def test_remove_service_service_id_not_found( mock_get2 = requests_mock.get( backend2 + "/services/wmts-foo", status_code=404, - # json={ - # "id": "936DA01F-9ABD-4D9D-80C7-02AF85C822A8", - # "code": "ServiceNotFound", - # "message": "Service 'wmts-foo' does not exist.", - # "url": "https://example.openeo.org/docs/errors/SampleError" - # } ) + # The aggregator should not execute a HTTP patch, so we check that it does not call these two. + mock_patch1 = requests_mock.patch( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=204 + ) + mock_patch2 = requests_mock.patch( + backend2 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=204 # deliberately avoid 404 so we know 404 comes from aggregator. + ) + comp_version = ComparableVersion(api_tester.api_version) + process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} + if comp_version < ComparableVersion((1, 0, 0)): + json_payload = {"process_graph": process_graph} + else: + json_payload = {"process": {"process_graph": process_graph}} - resp = api_tester.delete("/services/wmts-foo") + resp = api_tester.patch("/services/wmts-foo", json=json_payload) assert resp.status_code == 404 - # Make sure the aggregator asked the backend to remove the service. + # Verify that the aggregator did not try to call patch on the backend. + assert not mock_patch1.called + assert not mock_patch2.called + # Verify that the aggregator asked the backend to remove the service. assert mock_get1.called assert mock_get2.called From 386b2ce2ee8ab68caa40b740fe92e265cfa3496c Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Thu, 10 Nov 2022 15:05:54 +0100 Subject: [PATCH 16/26] Removing .flake8 which was added accidentally --- .flake8 | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 9292294a..00000000 --- a/.flake8 +++ /dev/null @@ -1,8 +0,0 @@ -[flake8] -# extend-ignore = E203 -exclude = - # No need to traverse our git directory - .git, - # There's no value in checking cache directories - __pycache__, -max-line-length=120 From 667bfa81a11d23f6d4c89ba6bc33738f535b1788 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Thu, 10 Nov 2022 15:51:16 +0100 Subject: [PATCH 17/26] Issue #78 Fix: methods for secondary services were not using the user_id --- src/openeo_aggregator/backend.py | 100 +++++++------- tests/test_backend.py | 217 ++++++++++++++++++------------- 2 files changed, 177 insertions(+), 140 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 70a92356..ada5dee2 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -824,7 +824,6 @@ def merge(services, to_add): # Collect all services from the backends. for con in self._backends: - # with con.authenticated_from_request(flask.request): with con.authenticated_from_request(request=flask.request, user=User(user_id)): services_json = None try: @@ -844,13 +843,14 @@ def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: # TODO: can there ever be a service with the same ID in multiple back-ends? (For the same user) for con in self._backends: - try: - service_json = con.get(f"/services/{service_id}").json() - except Exception as e: - _log.debug(f"No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) - continue - else: - return ServiceMetadata.from_dict(service_json) + with con.authenticated_from_request(request=flask.request, user=User(user_id)): + try: + service_json = con.get(f"/services/{service_id}").json() + except Exception as e: + _log.debug(f"No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) + continue + else: + return ServiceMetadata.from_dict(service_json) raise ServiceNotFoundException(service_id) @@ -868,70 +868,72 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a process_graph = self._processing.preprocess_process_graph(process_graph, backend_id=backend_id) con = self._backends.get_connection(backend_id) - try: - # create_service can raise ServiceUnsupportedException and OpenEOApiException. - service = con.create_service(graph=process_graph, type=service_type) + with con.authenticated_from_request(request=flask.request, user=User(user_id)): + try: + # create_service can raise ServiceUnsupportedException and OpenEOApiException. + service = con.create_service(graph=process_graph, type=service_type) - # TODO: This exception handling was copy-pasted. What do we actually need here? - except OpenEoApiError as e: - for exc_class in [ProcessGraphMissingException, ProcessGraphInvalidException]: - if e.code == exc_class.code: - raise exc_class - raise OpenEOApiException(f"Failed to create secondary service on backend {backend_id!r}: {e!r}") - except (OpenEoRestError, OpenEoClientException) as e: - raise OpenEOApiException(f"Failed to create secondary service on backend {backend_id!r}: {e!r}") + # TODO: This exception handling was copy-pasted. What do we actually need here? + except OpenEoApiError as e: + for exc_class in [ProcessGraphMissingException, ProcessGraphInvalidException]: + if e.code == exc_class.code: + raise exc_class + raise OpenEOApiException(f"Failed to create secondary service on backend {backend_id!r}: {e!r}") + except (OpenEoRestError, OpenEoClientException) as e: + raise OpenEOApiException(f"Failed to create secondary service on backend {backend_id!r}: {e!r}") - return service.service_id + return service.service_id - def _find_connection_with_service_id(self, service_id: str) -> BackendConnection: + def _find_connection_with_service_id(self, user_id: str, service_id: str) -> Optional[BackendConnection]: """Get connection for the backend that contains the service, return None if not found.""" # Search all services on the backends. for con in self._backends: - try: - _ = con.get(f"/services/{service_id}") - except OpenEoApiError as e: - if e.http_status_code == 404: - # Expected error - _log.debug(f"No service with ID={service_id!r} in backend with ID={con.id!r}: {e!r}", exc_info=True) - continue - else: + with con.authenticated_from_request(request=flask.request, user=User(user_id)): + try: + _ = con.get(f"/services/{service_id}") + except OpenEoApiError as e: + if e.http_status_code == 404: + # Expected error + _log.debug(f"No service with ID={service_id!r} in backend with ID={con.id!r}: {e!r}", exc_info=True) + continue + else: + _log.warning(f"Failed to get service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + raise e + except Exception as e: _log.warning(f"Failed to get service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise e - except Exception as e: - _log.warning(f"Failed to get service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) - raise e - else: - return con - + else: + return con return None def remove_service(self, user_id: str, service_id: str) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" # TODO: user_id is not used, how to authenticate when we use the BackendConnection? - con = self._find_connection_with_service_id(service_id) + con = self._find_connection_with_service_id(user_id, service_id) if not con: raise ServiceNotFoundException(service_id) - try: - con.delete(f"/services/{service_id}", expected_status=204) - except (OpenEoApiError, OpenEOApiException) as e: - # TODO: maybe we should just let these exception straight go to the caller without logging it here. - # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. - _log.warning(f"Failed to delete service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) - raise - except Exception as e: - _log.warning(f"Failed to delete service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) - raise OpenEOApiException( - f"Failed to delete service {service_id!r} on backend {con.id!r}: {e!r}" - ) from e + with con.authenticated_from_request(request=flask.request, user=User(user_id)): + try: + con.delete(f"/services/{service_id}", expected_status=204) + except (OpenEoApiError, OpenEOApiException) as e: + # TODO: maybe we should just let these exception straight go to the caller without logging it here. + # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. + _log.warning(f"Failed to delete service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + raise + except Exception as e: + _log.warning(f"Failed to delete service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + raise OpenEOApiException( + f"Failed to delete service {service_id!r} on backend {con.id!r}: {e!r}" + ) from e def update_service(self, user_id: str, service_id: str, process_graph: dict) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/update-service""" # TODO: user_id is not used, how to authenticate when we use the BackendConnection? - con = self._find_connection_with_service_id(service_id) + con = self._find_connection_with_service_id(user_id, service_id) if not con: raise ServiceNotFoundException(service_id) diff --git a/tests/test_backend.py b/tests/test_backend.py index a9358779..e1b1799e 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -239,14 +239,13 @@ def test_list_services_simple( """Given 2 backends but only 1 backend has a single service, then the aggregator returns that 1 service's metadata. """ - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - services1 = {"services": [service_metadata_wmts_foo.prepare_for_json()], "links": []} services2 = {} - requests_mock.get(backend1 + "/services", json=services1, headers=headers) - requests_mock.get(backend2 + "/services", json=services2, headers=headers) + requests_mock.get(backend1 + "/services", json=services1) + requests_mock.get(backend2 + "/services", json=services2) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): actual_services = abe_implementation.list_services(user_id=TEST_USER) @@ -264,14 +263,14 @@ def test_list_services_merged( service_metadata_wmts_foo, service_metadata_wms_bar ): """Given 2 backends with each 1 service, then the aggregator lists both services.""" - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER services1 = {"services": [service_metadata_wmts_foo.prepare_for_json()], "links": []} services2 = {"services": [service_metadata_wms_bar.prepare_for_json()], "links": []} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): actual_services = abe_implementation.list_services(user_id=TEST_USER) @@ -284,9 +283,6 @@ def test_list_services_merged_multiple( service_metadata_wmts_foo, service_metadata_wms_bar ): """Given multiple services across 2 backends, the aggregator lists all service types from all backends.""" - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - services1 = { "services": [{ "id": "wms-nvdi", @@ -333,6 +329,8 @@ def test_list_services_merged_multiple( abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): actual_services = abe_implementation.list_services(user_id=TEST_USER) @@ -349,7 +347,7 @@ def test_list_services_merged_multiple( assert sorted(actual_services) == sorted(expected_services) def test_service_info_succeeds( - self, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" @@ -358,15 +356,20 @@ def test_service_info_succeeds( abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER # Check the expected metadata on *both* of the services. - actual_service1 = abe_implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") - assert actual_service1 == service_metadata_wmts_foo - actual_service2 = abe_implementation.service_info(user_id=TEST_USER, service_id="wms-bar") - assert actual_service2 == service_metadata_wms_bar + with flask_app.test_request_context(headers=headers): + actual_service1 = abe_implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") + assert actual_service1 == service_metadata_wmts_foo + + with flask_app.test_request_context(headers=headers): + actual_service2 = abe_implementation.service_info(user_id=TEST_USER, service_id="wms-bar") + assert actual_service2 == service_metadata_wms_bar def test_service_info_wrong_id( - self, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" @@ -376,11 +379,16 @@ def test_service_info_wrong_id( abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - with pytest.raises(ServiceNotFoundException): - abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") + with flask_app.test_request_context(headers=headers): + with pytest.raises(ServiceNotFoundException): + abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") - def test_create_service_succeeds(self, api_tester, multi_backend_connection, config, backend1, requests_mock): + def test_create_service_succeeds( + self, flask_app, api_tester, multi_backend_connection, config, backend1, requests_mock + ): """When it gets a correct params for a new service, it successfully creates it.""" # Set up responses for creating the service in backend 1 @@ -398,19 +406,22 @@ def test_create_service_succeeds(self, api_tester, multi_backend_connection, con ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - actual_openeo_id = abe_implementation.create_service( - user_id=TEST_USER, - process_graph=process_graph, - service_type="WMTS", - api_version=api_tester.api_version, - configuration={} - ) - assert actual_openeo_id == expected_openeo_id + with flask_app.test_request_context(headers=headers): + actual_openeo_id = abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_tester.api_version, + configuration={} + ) + assert actual_openeo_id == expected_openeo_id @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) def test_create_service_backend_raises_openeoapiexception( - self, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, exception_class + self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, exception_class ): """When the backend raises a general exception the aggregator raises an OpenEOApiException.""" @@ -424,21 +435,24 @@ def test_create_service_backend_raises_openeoapiexception( exc=exception_class("Some server error"), ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - with pytest.raises(OpenEOApiException): - abe_implementation.create_service( - user_id=TEST_USER, - process_graph=process_graph, - service_type="WMTS", - api_version=api_tester.api_version, - configuration={} - ) + with flask_app.test_request_context(headers=headers): + with pytest.raises(OpenEOApiException): + abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_tester.api_version, + configuration={} + ) @pytest.mark.parametrize("exception_class", [ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException] ) def test_create_service_backend_reraises( - self, api_tester, multi_backend_connection, config, backend1, requests_mock, exception_class + self, flask_app, api_tester, multi_backend_connection, config, backend1, requests_mock, exception_class ): """When the backend raises exception types that indicate client error / bad input data, the aggregator raises and OpenEOApiException. @@ -453,19 +467,23 @@ def test_create_service_backend_reraises( exc=exception_class("Some server error") ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - # These exception types should be re-raised, not become an OpenEOApiException. - with pytest.raises(exception_class): - abe_implementation.create_service( - user_id=TEST_USER, - process_graph=process_graph, - service_type="WMTS", - api_version=api_tester.api_version, - configuration={} - ) + with flask_app.test_request_context(headers=headers): + # These exception types should be re-raised, not become an OpenEOApiException. + with pytest.raises(exception_class): + abe_implementation.create_service( + user_id=TEST_USER, + process_graph=process_graph, + service_type="WMTS", + api_version=api_tester.api_version, + configuration={} + ) def test_remove_service_succeeds( - self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, flask_app, api_tester, multi_backend_connection, config, + backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" @@ -482,18 +500,22 @@ def test_remove_service_succeeds( ) mock_delete = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + with flask_app.test_request_context(headers=headers): + abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") - # Make sure the aggregator asked the backend to remove the service. - assert mock_delete.called + # Make sure the aggregator asked the backend to remove the service. + assert mock_delete.called - # Check the other mocks were called too, just to be sure. - assert mock_get1.called - assert mock_get2.called + # Check the other mocks were called too, just to be sure. + assert mock_get1.called + assert mock_get2.called def test_remove_service_service_id_not_found( - self, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, flask_app, api_tester, multi_backend_connection, config, + backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" @@ -518,18 +540,21 @@ def test_remove_service_service_id_not_found( status_code=204 ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - with pytest.raises(ServiceNotFoundException): - abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + with flask_app.test_request_context(headers=headers): + with pytest.raises(ServiceNotFoundException): + abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") - assert not mock_delete1.called - assert not mock_delete2.called - # Check the other mocks were called too, just to be sure. - assert mock_get1.called - assert mock_get2.called + assert not mock_delete1.called + assert not mock_delete2.called + # Check the other mocks were called too, just to be sure. + assert mock_get1.called + assert mock_get2.called def test_remove_service_backend_response_is_an_error_status( - self, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo + self, flask_app, api_tester, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" @@ -541,17 +566,20 @@ def test_remove_service_backend_response_is_an_error_status( ) requests_mock.delete(backend1 + "/services/wmts-foo", status_code=500) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - with pytest.raises(OpenEoApiError) as e: - abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + with flask_app.test_request_context(headers=headers): + with pytest.raises(OpenEoApiError) as e: + abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") - # If the backend reports HTTP 400/500, we would expect the same status code from the aggregator. - # TODO: Statement above is an assumption. Is that really what we expect? - assert e.value.http_status_code == 500 + # If the backend reports HTTP 400/500, we would expect the same status code from the aggregator. + # TODO: Statement above is an assumption. Is that really what we expect? + assert e.value.http_status_code == 500 # TODO: this test still fails with API version 1.0.0 def test_update_service_succeeds( - self, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" @@ -574,26 +602,30 @@ def test_update_service_succeeds( ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + with flask_app.test_request_context(headers=headers): + abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) - # # Make sure the aggregator asked the backend to remove the service. - assert mock_patch.called - # TODO: I am not too sure this json payload is correct. Check with codebases of other backend drivers. - comp_api_version = ComparableVersion(api_tester.api_version) - if comp_api_version < ComparableVersion((1, 0, 0)): - expected_process = {"process_graph": process_graph_after} - else: - expected_process = {"process": process_graph_after} - - assert mock_patch.last_request.json() == expected_process + # # Make sure the aggregator asked the backend to remove the service. + assert mock_patch.called + # TODO: I am not too sure this json payload is correct. Check with codebases of other backend drivers. + comp_api_version = ComparableVersion(api_tester.api_version) + if comp_api_version < ComparableVersion((1, 0, 0)): + expected_process = {"process_graph": process_graph_after} + else: + expected_process = {"process": {"process_graph": process_graph_after}} - # Check the other mocks were called too, just to be sure. - assert mock_get1.called - assert mock_get2.called + assert mock_patch.last_request.json() == expected_process + + # Check the other mocks were called too, just to be sure. + assert mock_get1.called + assert mock_get2.called def test_update_service_service_id_not_found( - self, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, flask_app, api_tester, multi_backend_connection, config, + backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" @@ -620,15 +652,18 @@ def test_update_service_service_id_not_found( ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - with pytest.raises(ServiceNotFoundException): - abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) - - assert not mock_patch1.called - assert not mock_patch2.called - # Check the other mocks were called too, just to be sure. - assert mock_get1.called - assert mock_get2.called + with flask_app.test_request_context(headers=headers): + with pytest.raises(ServiceNotFoundException): + abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + + assert not mock_patch1.called + assert not mock_patch2.called + # Check the other mocks were called too, just to be sure. + assert mock_get1.called + assert mock_get2.called class TestInternalCollectionMetadata: From 66f005ab4040a0e97e2d339b0a5be601f051844d Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Thu, 10 Nov 2022 17:30:57 +0100 Subject: [PATCH 18/26] Issue #78 Add user authentication on update_service --- src/openeo_aggregator/backend.py | 40 +++++++++++-------------- tests/test_backend.py | 34 +++++++++++++++++++-- tests/test_views.py | 51 +++++++++++++++++++++++++++++--- 3 files changed, 97 insertions(+), 28 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index f4eb2065..1b44b569 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -660,7 +660,6 @@ def merge(formats: dict, to_add: dict): def list_services(self, user_id: str) -> List[ServiceMetadata]: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-services""" - # TODO: user_id is not used, how to authenticate when we use the BackendConnection? all_services = [] def merge(services, to_add): @@ -687,7 +686,6 @@ def merge(services, to_add): def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-service""" - # TODO: user_id is not used, how to authenticate when we use the BackendConnection? # TODO: can there ever be a service with the same ID in multiple back-ends? (For the same user) for con in self._backends: @@ -707,7 +705,6 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a """ https://openeo.org/documentation/1.0/developers/api/reference.html#operation/create-service """ - # TODO: user_id is not used, how to authenticate when we use the BackendConnection? # TODO: configuration is not used. What to do with it? backend_id = self._processing.get_backend_for_process_graph( @@ -757,7 +754,6 @@ def _find_connection_with_service_id(self, user_id: str, service_id: str) -> Opt def remove_service(self, user_id: str, service_id: str) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" - # TODO: user_id is not used, how to authenticate when we use the BackendConnection? con = self._find_connection_with_service_id(user_id, service_id) if not con: @@ -779,29 +775,29 @@ def remove_service(self, user_id: str, service_id: str) -> None: def update_service(self, user_id: str, service_id: str, process_graph: dict) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/update-service""" - # TODO: user_id is not used, how to authenticate when we use the BackendConnection? con = self._find_connection_with_service_id(user_id, service_id) if not con: raise ServiceNotFoundException(service_id) - api_version = self._backends.api_version - try: - if api_version < ComparableVersion((1, 0, 0)): - json = {"process_graph": process_graph} - else: - json = {"process": {"process_graph": process_graph}} - con.patch(f"/services/{service_id}", json=json, expected_status=204) - except (OpenEoApiError, OpenEOApiException) as e: - # TODO: maybe we should just let these exception straight go to the caller without logging it here. - # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. - _log.warning(f"Failed to update service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) - raise - except Exception as e: - _log.warning(f"Failed to update service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) - raise OpenEOApiException( - f"Failed to update service {service_id!r} from {con.id!r}: {e!r}" - ) from e + with con.authenticated_from_request(request=flask.request, user=User(user_id)): + api_version = self._backends.api_version + try: + if api_version < ComparableVersion((1, 0, 0)): + json = {"process_graph": process_graph} + else: + json = {"process": {"process_graph": process_graph}} + con.patch(f"/services/{service_id}", json=json, expected_status=204) + except (OpenEoApiError, OpenEOApiException) as e: + # TODO: maybe we should just let these exception straight go to the caller without logging it here. + # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. + _log.warning(f"Failed to update service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + raise + except Exception as e: + _log.warning(f"Failed to update service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + raise OpenEOApiException( + f"Failed to update service {service_id!r} from {con.id!r}: {e!r}" + ) from e class AggregatorBackendImplementation(OpenEoBackendImplementation): diff --git a/tests/test_backend.py b/tests/test_backend.py index 4414a3e4..d261b336 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -583,9 +583,10 @@ def test_update_service_succeeds( ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" - # Also test that it can skip backends that don't have the service set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + + # Also test that it can skip backends that don't have the service mock_get1 = requests_mock.get( backend1 + "/services/wmts-foo", status_code=404 @@ -629,7 +630,6 @@ def test_update_service_service_id_not_found( ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" - # Also test that it can skip backends that don't have the service set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) mock_get1 = requests_mock.get( @@ -665,6 +665,36 @@ def test_update_service_service_id_not_found( assert mock_get1.called assert mock_get2.called + def test_update_service_backend_response_is_an_error_status( + self, flask_app, api_tester, multi_backend_connection, config, + backend1, backend2, requests_mock, service_metadata_wmts_foo + ): + """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" + + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + mock_patch = requests_mock.patch( + backend1 + "/services/wmts-foo", + status_code=500, + ) + abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER + + with flask_app.test_request_context(headers=headers): + with pytest.raises(OpenEoApiError) as e: + abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + + assert e.value.http_status_code == 500 + assert mock_patch.called + + class TestInternalCollectionMetadata: diff --git a/tests/test_views.py b/tests/test_views.py index 798f763f..cf47b6d8 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1673,13 +1673,13 @@ def test_remove_service_succeeds( set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) - # Also test that it can skip backends that don't have the service - requests_mock.get( + # Also test that it skips backends that don't have the service2 + mock_get1 = requests_mock.get( backend1 + "/services/wmts-foo", status_code=404 ) # Delete should succeed in backend2 so service should be present first. - requests_mock.get( + mock_get2 = requests_mock.get( backend2 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json(), status_code=200 @@ -1691,6 +1691,9 @@ def test_remove_service_succeeds( assert resp.status_code == 204 # Make sure the aggregator asked the backend to remove the service. assert mock_delete.called + # Verify the aggregator did query the backends to find the service. + assert mock_get1.called + assert mock_get1.called def test_remove_service_service_id_not_found( self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo @@ -1764,7 +1767,7 @@ def test_update_service_service_succeeds( set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) - # Also test that it can skip backends that don't have the service + # Also test that it skips backends that don't have the service. mock_get1 = requests_mock.get( backend1 + "/services/wmts-foo", status_code=404 @@ -1844,6 +1847,46 @@ def test_update_service_service_id_not_found( assert mock_get1.called assert mock_get2.called + @pytest.mark.parametrize("backend_http_status", [400, 500]) + def test_update_service_backend_response_is_an_error_status( + self, flask_app, api_tester, multi_backend_connection, config, + backend1, backend2, requests_mock, service_metadata_wmts_foo, + backend_http_status + ): + """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" + api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + + requests_mock.get( + backend1 + "/services/wmts-foo", + json=service_metadata_wmts_foo.prepare_for_json(), + status_code=200 + ) + mock_patch = requests_mock.patch( + backend1 + "/services/wmts-foo", + status_code=backend_http_status, + json={ + "id": "936DA01F-9ABD-4D9D-80C7-02AF85C822A8", + "code": "ErrorUpdatingService", + "message": "Service 'wmts-foo' could not be updated.", + "url": "https://example.openeo.org/docs/errors/SampleError" + } + ) + + comp_version = ComparableVersion(api_tester.api_version) + process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} + if comp_version < ComparableVersion((1, 0, 0)): + json_payload = {"process_graph": process_graph} + else: + json_payload = {"process": {"process_graph": process_graph}} + + resp = api_tester.patch("/services/wmts-foo", json=json_payload) + + assert resp.status_code == backend_http_status + assert mock_patch.called + assert mock_patch.last_request.json() == json_payload + class TestResilience: From c5ffaf1fff3c45e8dcf4679ed914fd7ebc7862f7 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 14 Nov 2022 09:54:15 +0100 Subject: [PATCH 19/26] Issue #78, code review, doing some small corrections first --- tests/conftest.py | 6 ++---- tests/test_backend.py | 7 +++---- tests/test_views.py | 1 - 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2cd61654..ea0e30af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture -def backend1(requests_mock,) -> str: +def backend1(requests_mock) -> str: domain = "https://b1.test/v1" # TODO: how to work with different API versions? requests_mock.get(domain + "/", json={"api_version": "1.0.0"}) @@ -34,9 +34,7 @@ def backend2(requests_mock) -> str: # as "lib_requests_mock": make a distinction with the pytest fixture that has the same name -import requests_mock as lib_requests_mock - -def set_backend_to_api_version(requests_mock: lib_requests_mock.Mocker, domain: str, api_version: str) -> str: +def set_backend_to_api_version(requests_mock, domain: str, api_version: str) -> str: """Helper function to make the backend connection use the expected API version.""" # TODO: would like a nicer solution to make the backend fixtures match the expected API version. diff --git a/tests/test_backend.py b/tests/test_backend.py index d261b336..2e08fede 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,5 +1,4 @@ -from sys import implementation -from datetime import datetime +import datetime as dt import pytest @@ -215,7 +214,7 @@ def service_metadata_wmts_foo(self): configuration={"version": "0.5.8"}, attributes={}, title="Test WMTS service", - created=datetime(2020, 4, 9, 15, 5, 8) + created=dt.datetime(2020, 4, 9, 15, 5, 8) ) @pytest.fixture @@ -229,7 +228,7 @@ def service_metadata_wms_bar(self): configuration={"version": "0.5.8"}, attributes={}, title="Test WMS service", - created=datetime(2022, 2, 1, 13, 30, 3) + created=dt.datetime(2022, 2, 1, 13, 30, 3) ) def test_list_services_simple( diff --git a/tests/test_views.py b/tests/test_views.py index cf47b6d8..91ecf741 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1,5 +1,4 @@ import logging -import logging import re from typing import Tuple, List From 304b6ecb5d4b75dce7803a2c4d65aeeec5fed25d Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 14 Nov 2022 11:57:22 +0100 Subject: [PATCH 20/26] Issue #78, code review: removing test coverage for API version 0.4 --- src/openeo_aggregator/backend.py | 6 +- tests/conftest.py | 29 ---- tests/test_backend.py | 99 +++++++------- tests/test_views.py | 219 +++++++------------------------ 4 files changed, 95 insertions(+), 258 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 1b44b569..ef8bb9fa 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -781,12 +781,8 @@ def update_service(self, user_id: str, service_id: str, process_graph: dict) -> raise ServiceNotFoundException(service_id) with con.authenticated_from_request(request=flask.request, user=User(user_id)): - api_version = self._backends.api_version try: - if api_version < ComparableVersion((1, 0, 0)): - json = {"process_graph": process_graph} - else: - json = {"process": {"process_graph": process_graph}} + json = {"process": {"process_graph": process_graph}} con.patch(f"/services/{service_id}", json=json, expected_status=204) except (OpenEoApiError, OpenEOApiException) as e: # TODO: maybe we should just let these exception straight go to the caller without logging it here. diff --git a/tests/conftest.py b/tests/conftest.py index ea0e30af..39c7dd63 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -145,39 +145,10 @@ def backend_implementation(flask_app) -> AggregatorBackendImplementation: return flask_app.config["OPENEO_BACKEND_IMPLEMENTATION"] -@pytest.fixture(params=["0.4.0", "1.0.0"]) -def api_version_fixture(request): - """To go through all relevant API versions""" - return request.param - - -def get_api_tester(flask_app, api_version) -> ApiTester: - return ApiTester(api_version=api_version, client=flask_app.test_client()) - - -def get_api040(flask_app: flask.Flask) -> ApiTester: - return ApiTester(api_version="0.4.0", client=flask_app.test_client()) - - def get_api100(flask_app: flask.Flask) -> ApiTester: return ApiTester(api_version="1.0.0", client=flask_app.test_client()) -@pytest.fixture -def api_tester(flask_app, api_version_fixture) -> ApiTester: - """Get an ApiTester for each version. - - Useful when it easy to test several API versions with (mostly) the same test code. - But when the difference is too big, just keep it simple and write separate tests. - """ - return get_api_tester(flask_app, api_version_fixture) - - -@pytest.fixture -def api040(flask_app: flask.Flask) -> ApiTester: - return get_api040(flask_app) - - @pytest.fixture def api100(flask_app: flask.Flask) -> ApiTester: return get_api100(flask_app) diff --git a/tests/test_backend.py b/tests/test_backend.py index 2e08fede..9b717b1e 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -13,7 +13,6 @@ from openeo_driver.users.oidc import OidcProvider from openeo_driver.users.auth import HttpAuthHandler from openeo_driver.errors import ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException -from openeo.capabilities import ComparableVersion from openeo.rest import OpenEoApiError, OpenEoRestError from .conftest import DEFAULT_MEMOIZER_CONFIG, set_backend_to_api_version @@ -232,7 +231,7 @@ def service_metadata_wms_bar(self): ) def test_list_services_simple( - self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """Given 2 backends but only 1 backend has a single service, then the aggregator @@ -243,7 +242,7 @@ def test_list_services_simple( requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -258,7 +257,7 @@ def test_list_services_simple( assert actual_services == expected_services def test_list_services_merged( - self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """Given 2 backends with each 1 service, then the aggregator lists both services.""" @@ -268,7 +267,7 @@ def test_list_services_merged( requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -278,7 +277,7 @@ def test_list_services_merged( assert sorted(actual_services) == sorted(expected_services) def test_list_services_merged_multiple( - self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """Given multiple services across 2 backends, the aggregator lists all service types from all backends.""" @@ -328,7 +327,7 @@ def test_list_services_merged_multiple( abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -346,7 +345,7 @@ def test_list_services_merged_multiple( assert sorted(actual_services) == sorted(expected_services) def test_service_info_succeeds( - self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" @@ -355,7 +354,7 @@ def test_service_info_succeeds( abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER # Check the expected metadata on *both* of the services. @@ -368,7 +367,7 @@ def test_service_info_succeeds( assert actual_service2 == service_metadata_wms_bar def test_service_info_wrong_id( - self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" @@ -378,7 +377,7 @@ def test_service_info_wrong_id( abe_implementation = AggregatorBackendImplementation( backends=multi_backend_connection, config=config ) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -386,12 +385,12 @@ def test_service_info_wrong_id( abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") def test_create_service_succeeds( - self, flask_app, api_tester, multi_backend_connection, config, backend1, requests_mock + self, flask_app, api100, multi_backend_connection, config, backend1, requests_mock ): """When it gets a correct params for a new service, it successfully creates it.""" # Set up responses for creating the service in backend 1 - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") expected_openeo_id = "wmts-foo" location_backend_1 = backend1 + "/services/" + expected_openeo_id process_graph = {"foo": {"process_id": "foo", "arguments": {}}} @@ -405,7 +404,7 @@ def test_create_service_succeeds( ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -413,28 +412,28 @@ def test_create_service_succeeds( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", - api_version=api_tester.api_version, + api_version="1.0.0", configuration={} ) assert actual_openeo_id == expected_openeo_id @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) def test_create_service_backend_raises_openeoapiexception( - self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, exception_class + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, exception_class ): """When the backend raises a general exception the aggregator raises an OpenEOApiException.""" # Set up responses for creating the service in backend 1: # This time the backend raises an error, one that will be reported as a OpenEOApiException. - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", exc=exception_class("Some server error"), ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -443,7 +442,7 @@ def test_create_service_backend_raises_openeoapiexception( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", - api_version=api_tester.api_version, + api_version="1.0.0", configuration={} ) @@ -451,7 +450,7 @@ def test_create_service_backend_raises_openeoapiexception( [ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException] ) def test_create_service_backend_reraises( - self, flask_app, api_tester, multi_backend_connection, config, backend1, requests_mock, exception_class + self, flask_app, api100, multi_backend_connection, config, backend1, requests_mock, exception_class ): """When the backend raises exception types that indicate client error / bad input data, the aggregator raises and OpenEOApiException. @@ -459,14 +458,14 @@ def test_create_service_backend_reraises( # Set up responses for creating the service in backend 1 # This time the backend raises an error, one that will simply be re-raised/passed on as it is. - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", exc=exception_class("Some server error") ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -476,12 +475,12 @@ def test_create_service_backend_reraises( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", - api_version=api_tester.api_version, + api_version="1.0.0", configuration={} ) def test_remove_service_succeeds( - self, flask_app, api_tester, multi_backend_connection, config, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" @@ -499,7 +498,7 @@ def test_remove_service_succeeds( ) mock_delete = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -513,7 +512,7 @@ def test_remove_service_succeeds( assert mock_get2.called def test_remove_service_service_id_not_found( - self, flask_app, api_tester, multi_backend_connection, config, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" @@ -539,7 +538,7 @@ def test_remove_service_service_id_not_found( status_code=204 ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -553,7 +552,7 @@ def test_remove_service_service_id_not_found( assert mock_get2.called def test_remove_service_backend_response_is_an_error_status( - self, flask_app, api_tester, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo + self, flask_app, api100, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" @@ -565,7 +564,7 @@ def test_remove_service_backend_response_is_an_error_status( ) requests_mock.delete(backend1 + "/services/wmts-foo", status_code=500) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -578,13 +577,13 @@ def test_remove_service_backend_response_is_an_error_status( # TODO: this test still fails with API version 1.0.0 def test_update_service_succeeds( - self, flask_app, api_tester, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) - + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") + # Also test that it can skip backends that don't have the service mock_get1 = requests_mock.get( backend1 + "/services/wmts-foo", @@ -602,21 +601,17 @@ def test_update_service_succeeds( ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) - # # Make sure the aggregator asked the backend to remove the service. + # Make sure the aggregator asked the backend to remove the service. assert mock_patch.called + # TODO: I am not too sure this json payload is correct. Check with codebases of other backend drivers. - comp_api_version = ComparableVersion(api_tester.api_version) - if comp_api_version < ComparableVersion((1, 0, 0)): - expected_process = {"process_graph": process_graph_after} - else: - expected_process = {"process": {"process_graph": process_graph_after}} - + expected_process = {"process": {"process_graph": process_graph_after}} assert mock_patch.last_request.json() == expected_process # Check the other mocks were called too, just to be sure. @@ -624,13 +619,13 @@ def test_update_service_succeeds( assert mock_get2.called def test_update_service_service_id_not_found( - self, flask_app, api_tester, multi_backend_connection, config, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") mock_get1 = requests_mock.get( backend1 + "/services/wmts-foo", status_code=404 @@ -651,7 +646,7 @@ def test_update_service_service_id_not_found( ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): @@ -665,13 +660,13 @@ def test_update_service_service_id_not_found( assert mock_get2.called def test_update_service_backend_response_is_an_error_status( - self, flask_app, api_tester, multi_backend_connection, config, + self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") requests_mock.get( backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json(), @@ -682,13 +677,13 @@ def test_update_service_backend_response_is_an_error_status( status_code=500, ) abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) - process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + new_process_graph = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): with pytest.raises(OpenEoApiError) as e: - abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=new_process_graph) assert e.value.http_status_code == 500 assert mock_patch.called diff --git a/tests/test_views.py b/tests/test_views.py index 91ecf741..dc0d6624 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -5,7 +5,6 @@ import pytest import requests -from openeo.capabilities import ComparableVersion from openeo.rest.connection import url_join from openeo.rest import OpenEoApiError, OpenEoRestError from openeo_aggregator.backend import AggregatorCollectionCatalog @@ -1350,7 +1349,7 @@ def service_metadata_wms_bar(self): # not setting "created": This is used to test creating a service. ) - def test_service_types_simple(self, api_tester, backend1, backend2, requests_mock): + def test_service_types_simple(self, api100, backend1, backend2, requests_mock): """Given 2 backends but only 1 backend has a single service, then the aggregator returns that 1 service's metadata. """ @@ -1378,10 +1377,10 @@ def test_service_types_simple(self, api_tester, backend1, backend2, requests_moc requests_mock.get(backend1 + "/service_types", json=single_service_type) requests_mock.get(backend2 + "/service_types", json=single_service_type) - resp = api_tester.get('/service_types').assert_status_code(200) + resp = api100.get('/service_types').assert_status_code(200) assert resp.json == single_service_type - def test_service_types_merging(self, api_tester, backend1, backend2, requests_mock): + def test_service_types_merging(self, api100, backend1, backend2, requests_mock): """Given 2 backends with each 1 service, then the aggregator lists both services.""" service_type_1 = { "WMTS": { @@ -1415,14 +1414,14 @@ def test_service_types_merging(self, api_tester, backend1, backend2, requests_mo requests_mock.get(backend1 + "/service_types", json=service_type_1) requests_mock.get(backend2 + "/service_types", json=service_type_2) - resp = api_tester.get("/service_types").assert_status_code(200) + resp = api100.get("/service_types").assert_status_code(200) actual_service_types = resp.json expected_service_types = dict(service_type_1) expected_service_types.update(service_type_2) assert actual_service_types == expected_service_types - def test_service_info_api100( + def test_service_info( self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" @@ -1458,80 +1457,18 @@ def test_service_info_api100( ) assert actual_service2 == service_metadata_wms_bar - def test_service_info_api040( - self, api040, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar - ): - """When it gets a correct service ID, it returns the expected ServiceMetadata.""" - - expected_service1 = service_metadata_wmts_foo - expected_service2 = service_metadata_wms_bar - requests_mock.get(backend1 + "/services/wmts-foo", json=expected_service1.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=expected_service2.prepare_for_json()) - api040.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) - - # Retrieve and verify the metadata for both services - # Here we compare attribute by attribute because in API version 0.4.0 the response - # has fewer properties, and some have a slightly different name and data structure. - resp = api040.get("/services/wmts-foo").assert_status_code(200) - assert expected_service1.id == resp.json["id"] - assert expected_service1.process["process_graph"] == resp.json["process_graph"] - assert expected_service1.url == resp.json["url"] - assert expected_service1.type == resp.json["type"] - assert expected_service1.title == resp.json["title"] - assert expected_service1.attributes == resp.json["attributes"] - - resp = api040.get("/services/wms-bar").assert_status_code(200) - assert expected_service2.id == resp.json["id"] - assert expected_service2.process["process_graph"] == resp.json["process_graph"] - assert expected_service2.url == resp.json["url"] - assert expected_service2.type == resp.json["type"] - assert expected_service2.title == resp.json["title"] - assert expected_service2.attributes == resp.json["attributes"] - def test_service_info_wrong_id( - self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar + self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a non-existent service ID, it returns HTTP Status 404, Not found.""" requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) - api_tester.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) - - api_tester.get("/services/doesnotexist").assert_status_code(404) - - def test_create_wmts_040(self, api040, requests_mock, backend1): - api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - - expected_openeo_id = 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' - # The aggregator MUST NOT point to the actual instance but to its own endpoint. - # This is handled by the openeo python driver in openeo_driver.views.services_post. - expected_location = "/openeo/0.4.0/services/" + expected_openeo_id - # However, backend1 must report its OWN location. - location_backend_1 = backend1 + "/services" + expected_openeo_id - - process_graph = {"foo": {"process_id": "foo", "arguments": {}}} - # The process_graph/process format is slightly different between api v0.4 and v1.0 - post_data = { - "type": 'WMTS', - "process_graph": process_graph, - "title": "My Service", - "description": "Service description" - } - - requests_mock.post( - backend1 + "/services", - headers={ - "OpenEO-Identifier": expected_openeo_id, - "Location": location_backend_1 - }, - status_code=201 - ) + api100.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) - resp = api040.post('/services', json=post_data).assert_status_code(201) - assert resp.headers['OpenEO-Identifier'] == expected_openeo_id - assert resp.headers['Location'] == expected_location + api100.get("/services/doesnotexist").assert_status_code(404) - def test_create_wmts_100(self, api100, requests_mock, backend1): + def test_create_wmts(self, api100, requests_mock, backend1): api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) expected_openeo_id = 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' @@ -1566,36 +1503,9 @@ def test_create_wmts_100(self, api100, requests_mock, backend1): assert resp.headers['OpenEO-Identifier'] == 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' assert resp.headers['Location'] == expected_location - # TODO: maybe testing specifically client error vs server error goes to far. It may be a bit too complicated. - # ProcessGraphMissingException and ProcessGraphInvalidException are well known reasons for a bad client request. - @pytest.mark.parametrize("exception_class", [ProcessGraphMissingException, ProcessGraphInvalidException]) - def test_create_wmts_reports_400_client_error_api040(self, api040, requests_mock, backend1, exception_class): - """When the backend raised an exception that we know represents incorrect input / client error, - then the aggregator's responds with an HTTP status code in the 400 range. - """ - api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - - process_graph = {"foo": {"process_id": "foo", "arguments": {}}} - # The process_graph/process format is slightly different between api v0.4 and v1.0 - post_data = { - "type": 'WMTS', - "process_graph": process_graph, - "title": "My Service", - "description": "Service description" - } - # TODO: In theory we should make the backend report a HTTP 400 status and then the aggregator - # should also report HTTP 400. But in fact that comes back as HTTP 500. - requests_mock.post( - backend1 + "/services", - exc=exception_class("Testing exception handling") - ) - - resp = api040.post('/services', json=post_data) - assert resp.status_code == 400 - # ProcessGraphMissingException and ProcessGraphInvalidException are well known reasons for a bad client request. @pytest.mark.parametrize("exception_class", [ProcessGraphMissingException, ProcessGraphInvalidException]) - def test_create_wmts_reports_400_client_error_api100(self, api100, requests_mock, backend1, exception_class): + def test_create_wmts_reports_400_client_error(self, api100, requests_mock, backend1, exception_class): api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) process_graph = {"foo": {"process_id": "foo", "arguments": {}}} @@ -1621,28 +1531,7 @@ def test_create_wmts_reports_400_client_error_api100(self, api100, requests_mock # OpenEoApiError, OpenEoRestError: more general errors we can expect to lead to a HTTP 500 server error. @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) - def test_create_wmts_reports_500_server_error_api040(self, api040, requests_mock, backend1, exception_class): - api040.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - - process_graph = {"foo": {"process_id": "foo", "arguments": {}}} - # The process_graph/process format is slightly different between api v0.4 and v1.0 - post_data = { - "type": 'WMTS', - "process_graph": process_graph, - "title": "My Service", - "description": "Service description" - } - requests_mock.post( - backend1 + "/services", - exc=exception_class("Testing exception handling") - ) - - resp = api040.post('/services', json=post_data) - assert resp.status_code == 500 - - # OpenEoApiError, OpenEoRestError: more general errors we can expect to lead to a HTTP 500 server error. - @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) - def test_create_wmts_reports_500_server_error_api100(self, api100, requests_mock, backend1, exception_class): + def test_create_wmts_reports_500_server_error(self, api100, requests_mock, backend1, exception_class): api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) process_graph = {"foo": {"process_id": "foo", "arguments": {}}} @@ -1665,12 +1554,12 @@ def test_create_wmts_reports_500_server_error_api100(self, api100, requests_mock assert resp.status_code == 500 def test_remove_service_succeeds( - self, api_tester, requests_mock, backend1, backend2, service_metadata_wmts_foo + self, api100, requests_mock, backend1, backend2, service_metadata_wmts_foo ): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Also test that it skips backends that don't have the service2 mock_get1 = requests_mock.get( @@ -1685,7 +1574,7 @@ def test_remove_service_succeeds( ) mock_delete = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) - resp = api_tester.delete("/services/wmts-foo") + resp = api100.delete("/services/wmts-foo") assert resp.status_code == 204 # Make sure the aggregator asked the backend to remove the service. @@ -1695,12 +1584,12 @@ def test_remove_service_succeeds( assert mock_get1.called def test_remove_service_service_id_not_found( - self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Neither backend has the service available, and the aggregator should detect this. mock_get1 = requests_mock.get( @@ -1717,7 +1606,7 @@ def test_remove_service_service_id_not_found( status_code=204, # deliberately avoid 404 so we know 404 comes from aggregator. ) - resp = api_tester.delete("/services/wmts-foo") + resp = api100.delete("/services/wmts-foo") assert resp.status_code == 404 # Verify the aggregator did not call the backend to remove the service. @@ -1727,12 +1616,12 @@ def test_remove_service_service_id_not_found( assert mock_get2.called def test_remove_service_backend_response_is_an_error_status( - self, api_tester, requests_mock, backend1, backend2, service_metadata_wmts_foo + self, api100, requests_mock, backend1, backend2, service_metadata_wmts_foo ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Will find it on the first backend, and it should skip the second backend so we don't add it to backend2. requests_mock.get( @@ -1751,7 +1640,7 @@ def test_remove_service_backend_response_is_an_error_status( } ) - resp = api_tester.delete("/services/wmts-foo") + resp = api100.delete("/services/wmts-foo") assert resp.status_code == 500 # Verify the aggregator effectively asked the backend to remove the service, @@ -1759,12 +1648,12 @@ def test_remove_service_backend_response_is_an_error_status( assert mock_delete.called def test_update_service_service_succeeds( - self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Also test that it skips backends that don't have the service. mock_get1 = requests_mock.get( @@ -1781,15 +1670,10 @@ def test_update_service_service_succeeds( json=service_metadata_wmts_foo.prepare_for_json(), status_code=204 ) - - comp_version = ComparableVersion(api_tester.api_version) process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} - if comp_version < ComparableVersion((1, 0, 0)): - json_payload = {"process_graph": process_graph} - else: - json_payload = {"process": {"process_graph": process_graph}} + json_payload = {"process": {"process_graph": process_graph}} - resp = api_tester.patch("/services/wmts-foo", json=json_payload) + resp = api100.patch("/services/wmts-foo", json=json_payload) assert resp.status_code == 204 # Make sure the aggregator asked the backend to update the service. @@ -1801,12 +1685,12 @@ def test_update_service_service_succeeds( assert mock_get2.called def test_update_service_service_id_not_found( - self, api_tester, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Neither backend has the service available, and the aggregator should detect this. mock_get1 = requests_mock.get( @@ -1829,14 +1713,10 @@ def test_update_service_service_id_not_found( json=service_metadata_wmts_foo.prepare_for_json(), status_code=204 # deliberately avoid 404 so we know 404 comes from aggregator. ) - comp_version = ComparableVersion(api_tester.api_version) process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} - if comp_version < ComparableVersion((1, 0, 0)): - json_payload = {"process_graph": process_graph} - else: - json_payload = {"process": {"process_graph": process_graph}} + json_payload = {"process": {"process_graph": process_graph}} - resp = api_tester.patch("/services/wmts-foo", json=json_payload) + resp = api100.patch("/services/wmts-foo", json=json_payload) assert resp.status_code == 404 # Verify that the aggregator did not try to call patch on the backend. @@ -1846,16 +1726,16 @@ def test_update_service_service_id_not_found( assert mock_get1.called assert mock_get2.called - @pytest.mark.parametrize("backend_http_status", [400, 500]) + # TODO: for now, not bothering with HTTP 400 in the backend. To be decided if this is necessary. + # @pytest.mark.parametrize("backend_http_status", [400, 500]) + @pytest.mark.parametrize("backend_http_status", [500]) def test_update_service_backend_response_is_an_error_status( - self, flask_app, api_tester, multi_backend_connection, config, - backend1, backend2, requests_mock, service_metadata_wmts_foo, - backend_http_status + self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo, backend_http_status ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" - api_tester.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, api_tester.api_version) - set_backend_to_api_version(requests_mock, backend2, api_tester.api_version) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + set_backend_to_api_version(requests_mock, backend1, "1.0.0") + set_backend_to_api_version(requests_mock, backend2, "1.0.0") requests_mock.get( backend1 + "/services/wmts-foo", @@ -1872,15 +1752,10 @@ def test_update_service_backend_response_is_an_error_status( "url": "https://example.openeo.org/docs/errors/SampleError" } ) - - comp_version = ComparableVersion(api_tester.api_version) process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} - if comp_version < ComparableVersion((1, 0, 0)): - json_payload = {"process_graph": process_graph} - else: - json_payload = {"process": {"process_graph": process_graph}} + json_payload = {"process": {"process_graph": process_graph}} - resp = api_tester.patch("/services/wmts-foo", json=json_payload) + resp = api100.patch("/services/wmts-foo", json=json_payload) assert resp.status_code == backend_http_status assert mock_patch.called From c0b14805c2fa0a1afa5c6b157457737a2dd98a6e Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 14 Nov 2022 15:03:49 +0100 Subject: [PATCH 21/26] Issue #78, corrected: methods from AggregatorSecondaryServices were not supposed to be added to AggregatorBackendImplementation --- src/openeo_aggregator/backend.py | 67 ++++++++------ tests/test_backend.py | 150 +++++++++++++++++-------------- 2 files changed, 124 insertions(+), 93 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index ef8bb9fa..f9283a81 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -620,6 +620,24 @@ def get_log_entries(self, job_id: str, user_id: str, offset: Optional[str] = Non return con.job(backend_job_id).logs(offset=offset) +class ServiceIdMapping: + """Mapping between aggregator service ids and backend job ids""" + + @staticmethod + def get_aggregator_service_id(backend_service_id: str, backend_id: str) -> str: + """Construct aggregator service id from given backend job id and backend id""" + return f"{backend_id}-{backend_service_id}" + + @classmethod + def parse_aggregator_service_id(cls, backends: MultiBackendConnection, aggregator_service_id: str) -> Tuple[str, str]: + """Given aggregator service id: extract backend service id and backend id""" + for prefix in [f"{con.id}-" for con in backends]: + if aggregator_service_id.startswith(prefix): + backend_id, backend_job_id = aggregator_service_id.split("-", maxsplit=1) + return backend_job_id, backend_id + raise ServiceNotFoundException(service_id=aggregator_service_id) + + class AggregatorSecondaryServices(SecondaryServices): """ Aggregator implementation of the Secondary Services "microservice" @@ -635,6 +653,18 @@ def __init__( self._backends = backends self._processing = processing + def _get_connection_and_backend_service_id( + self, + aggregator_service_id: str + ) -> Tuple[Union[BackendConnection, BackendConnection], str]: + backend_service_id, backend_id = ServiceIdMapping.parse_aggregator_service_id( + backends=self._backends, + aggregator_service_id=aggregator_service_id + ) + + con = self._backends.get_connection(backend_id) + return con, backend_service_id + def service_types(self) -> dict: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-service-types""" @@ -661,6 +691,7 @@ def merge(formats: dict, to_add: dict): def list_services(self, user_id: str) -> List[ServiceMetadata]: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-services""" + # TODO: use ServiceIdMapping to prepend all service IDs with their backend-id all_services = [] def merge(services, to_add): # For now ignore the links @@ -687,7 +718,17 @@ def merge(services, to_add): def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-service""" - # TODO: can there ever be a service with the same ID in multiple back-ends? (For the same user) + # con, backend_service_id = self._get_connection_and_backend_service_id(service_id) + # with con.authenticated_from_request(request=flask.request, user=User(user_id)): + # try: + # service_json = con.get(f"/services/{backend_service_id}").json() + # except Exception as e: + # _log.debug(f"Failed to get service with ID={backend_service_id} from backend with ID={con.id}: {e!r}", exc_info=True) + # raise + # else: + # service_json["id"] = ServiceIdMapping.get_aggregator_service_id(service_json["id"], con.id) + # return ServiceMetadata.from_dict(service_json) + for con in self._backends: with con.authenticated_from_request(request=flask.request, user=User(user_id)): try: @@ -980,27 +1021,3 @@ def postprocess_capabilities(self, capabilities: dict) -> dict: # TODO: standardize this field? capabilities["_partitioned_job_tracking"] = bool(self.batch_jobs.partitioned_job_tracker) return capabilities - - def service_types(self) -> dict: - return self.secondary_services.service_types() - - def list_services(self, user_id: str) -> List[ServiceMetadata]: - return self.secondary_services.list_services(user_id=user_id) - - def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: - return self.secondary_services.service_info(user_id=user_id, service_id=service_id) - - def create_service(self, user_id: str, process_graph: dict, service_type: str, api_version: str, - configuration: dict) -> str: - return self.secondary_services.create_service(user_id=user_id, process_graph=process_graph, - service_type=service_type, api_version=api_version, configuration=configuration) - - def remove_service(self, user_id: str, service_id: str) -> None: - """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" - self.secondary_services.remove_service(user_id=user_id, service_id=service_id) - - def update_service(self, user_id: str, service_id: str, process_graph: dict) -> None: - """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/update-service""" - self.secondary_services.update_service( - user_id=user_id, service_id=service_id, process_graph=process_graph - ) \ No newline at end of file diff --git a/tests/test_backend.py b/tests/test_backend.py index 9b717b1e..568988de 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -3,7 +3,8 @@ import pytest from openeo_aggregator.backend import AggregatorCollectionCatalog, AggregatorProcessing, \ - AggregatorBackendImplementation, _InternalCollectionMetadata, JobIdMapping + AggregatorBackendImplementation, _InternalCollectionMetadata, JobIdMapping, \ + AggregatorSecondaryServices from openeo_aggregator.caching import DictMemoizer from openeo_aggregator.testing import clock_mock from openeo_driver.backend import ServiceMetadata @@ -126,7 +127,9 @@ def test_file_formats_merging(self, multi_backend_connection, config, backend1, class TestAggregatorSecondaryServices: - def test_service_types_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): + def test_service_types_simple( + self, multi_backend_connection, config, catalog, backend1, backend2, requests_mock + ): """Given 2 backends and only 1 backend has a single service type, then the aggregator returns that 1 service type's metadata. """ @@ -152,14 +155,14 @@ def test_service_types_simple(self, multi_backend_connection, config, backend1, } } requests_mock.get(backend1 + "/service_types", json=single_service_type) - requests_mock.get(backend2 + "/service_types", json=single_service_type) - abe_implementation = AggregatorBackendImplementation( - backends=multi_backend_connection, config=config - ) - service_types = abe_implementation.service_types() + requests_mock.get(backend2 + "/service_types", json={}) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) + + service_types = implementation.service_types() assert service_types == single_service_type - def test_service_types_merging(self, multi_backend_connection, config, backend1, backend2, requests_mock): + def test_service_types_merging(self, multi_backend_connection, config, catalog, backend1, backend2, requests_mock): """Given 2 backends with each 1 service type, then the aggregator lists both service types.""" service_type_1 = { "WMTS": { @@ -192,11 +195,10 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, } requests_mock.get(backend1 + "/service_types", json=service_type_1) requests_mock.get(backend2 + "/service_types", json=service_type_2) - abe_implementation = AggregatorBackendImplementation( - backends=multi_backend_connection, config=config - ) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - actual_service_types = abe_implementation.service_types() + actual_service_types = implementation.service_types() expected_service_types = dict(service_type_1) expected_service_types.update(service_type_2) @@ -231,7 +233,7 @@ def service_metadata_wms_bar(self): ) def test_list_services_simple( - self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """Given 2 backends but only 1 backend has a single service, then the aggregator @@ -241,23 +243,25 @@ def test_list_services_simple( services2 = {} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): - actual_services = abe_implementation.list_services(user_id=TEST_USER) + actual_services = implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in services1 # (there is only one) for conversion to a ServiceMetadata. - the_service = services1["services"][0] - expected_services = [ - ServiceMetadata.from_dict(the_service) - ] + the_service = dict(services1["services"][0]) + + # TODO: prepend the backend's service_id with the backend_id + # the_service["id"] = "-wmts-foo" + expected_services = [ServiceMetadata.from_dict(the_service)] assert actual_services == expected_services def test_list_services_merged( - self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """Given 2 backends with each 1 service, then the aggregator lists both services.""" @@ -266,18 +270,20 @@ def test_list_services_merged( services2 = {"services": [service_metadata_wms_bar.prepare_for_json()], "links": []} requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): - actual_services = abe_implementation.list_services(user_id=TEST_USER) + actual_services = implementation.list_services(user_id=TEST_USER) expected_services = [service_metadata_wmts_foo, service_metadata_wms_bar] assert sorted(actual_services) == sorted(expected_services) def test_list_services_merged_multiple( - self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """Given multiple services across 2 backends, the aggregator lists all service types from all backends.""" @@ -324,14 +330,13 @@ def test_list_services_merged_multiple( } requests_mock.get(backend1 + "/services", json=services1) requests_mock.get(backend2 + "/services", json=services2) - abe_implementation = AggregatorBackendImplementation( - backends=multi_backend_connection, config=config - ) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): - actual_services = abe_implementation.list_services(user_id=TEST_USER) + actual_services = implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in # services1 (there is only one) for conversion to a ServiceMetadata. @@ -345,47 +350,45 @@ def test_list_services_merged_multiple( assert sorted(actual_services) == sorted(expected_services) def test_service_info_succeeds( - self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) - abe_implementation = AggregatorBackendImplementation( - backends=multi_backend_connection, config=config - ) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER # Check the expected metadata on *both* of the services. with flask_app.test_request_context(headers=headers): - actual_service1 = abe_implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") + actual_service1 = implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") assert actual_service1 == service_metadata_wmts_foo with flask_app.test_request_context(headers=headers): - actual_service2 = abe_implementation.service_info(user_id=TEST_USER, service_id="wms-bar") + actual_service2 = implementation.service_info(user_id=TEST_USER, service_id="wms-bar") assert actual_service2 == service_metadata_wms_bar def test_service_info_wrong_id( - self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) - abe_implementation = AggregatorBackendImplementation( - backends=multi_backend_connection, config=config - ) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): with pytest.raises(ServiceNotFoundException): - abe_implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") + implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") def test_create_service_succeeds( - self, flask_app, api100, multi_backend_connection, config, backend1, requests_mock + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock ): """When it gets a correct params for a new service, it successfully creates it.""" @@ -402,13 +405,13 @@ def test_create_service_succeeds( }, status_code=201 ) - - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): - actual_openeo_id = abe_implementation.create_service( + actual_openeo_id = implementation.create_service( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", @@ -419,7 +422,8 @@ def test_create_service_succeeds( @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) def test_create_service_backend_raises_openeoapiexception( - self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, exception_class + self, flask_app, api100, multi_backend_connection, config, catalog, + backend1, backend2, requests_mock, exception_class ): """When the backend raises a general exception the aggregator raises an OpenEOApiException.""" @@ -432,13 +436,14 @@ def test_create_service_backend_raises_openeoapiexception( backend1 + "/services", exc=exception_class("Some server error"), ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): with pytest.raises(OpenEOApiException): - abe_implementation.create_service( + implementation.create_service( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", @@ -450,7 +455,8 @@ def test_create_service_backend_raises_openeoapiexception( [ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException] ) def test_create_service_backend_reraises( - self, flask_app, api100, multi_backend_connection, config, backend1, requests_mock, exception_class + self, flask_app, api100, multi_backend_connection, config, catalog, + backend1, requests_mock, exception_class ): """When the backend raises exception types that indicate client error / bad input data, the aggregator raises and OpenEOApiException. @@ -464,14 +470,15 @@ def test_create_service_backend_reraises( backend1 + "/services", exc=exception_class("Some server error") ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): # These exception types should be re-raised, not become an OpenEOApiException. with pytest.raises(exception_class): - abe_implementation.create_service( + implementation.create_service( user_id=TEST_USER, process_graph=process_graph, service_type="WMTS", @@ -480,7 +487,7 @@ def test_create_service_backend_reraises( ) def test_remove_service_succeeds( - self, flask_app, api100, multi_backend_connection, config, + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" @@ -497,12 +504,13 @@ def test_remove_service_succeeds( status_code=200 ) mock_delete = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): - abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") # Make sure the aggregator asked the backend to remove the service. assert mock_delete.called @@ -512,7 +520,7 @@ def test_remove_service_succeeds( assert mock_get2.called def test_remove_service_service_id_not_found( - self, flask_app, api100, multi_backend_connection, config, + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" @@ -537,13 +545,14 @@ def test_remove_service_service_id_not_found( backend2 + "/services/wmts-foo", status_code=204 ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): with pytest.raises(ServiceNotFoundException): - abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") assert not mock_delete1.called assert not mock_delete2.called @@ -552,7 +561,8 @@ def test_remove_service_service_id_not_found( assert mock_get2.called def test_remove_service_backend_response_is_an_error_status( - self, flask_app, api100, multi_backend_connection, config, backend1, requests_mock, service_metadata_wmts_foo + self, flask_app, api100, multi_backend_connection, config, catalog, + backend1, requests_mock, service_metadata_wmts_foo ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" @@ -563,13 +573,14 @@ def test_remove_service_backend_response_is_an_error_status( status_code=200 ) requests_mock.delete(backend1 + "/services/wmts-foo", status_code=500) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): with pytest.raises(OpenEoApiError) as e: - abe_implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") # If the backend reports HTTP 400/500, we would expect the same status code from the aggregator. # TODO: Statement above is an assumption. Is that really what we expect? @@ -577,7 +588,8 @@ def test_remove_service_backend_response_is_an_error_status( # TODO: this test still fails with API version 1.0.0 def test_update_service_succeeds( - self, flask_app, api100, multi_backend_connection, config, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, flask_app, api100, multi_backend_connection, config, catalog, + backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" @@ -599,13 +611,14 @@ def test_update_service_succeeds( backend2 + "/services/wmts-foo", status_code=204, ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): - abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) # Make sure the aggregator asked the backend to remove the service. assert mock_patch.called @@ -619,8 +632,8 @@ def test_update_service_succeeds( assert mock_get2.called def test_update_service_service_id_not_found( - self, flask_app, api100, multi_backend_connection, config, - backend1, backend2, requests_mock, service_metadata_wmts_foo + self, flask_app, api100, multi_backend_connection, config, catalog, + backend1, backend2, requests_mock ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" @@ -644,14 +657,15 @@ def test_update_service_service_id_not_found( backend2 + "/services/wmts-foo", status_code=204, ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): with pytest.raises(ServiceNotFoundException): - abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) assert not mock_patch1.called assert not mock_patch2.called @@ -660,7 +674,7 @@ def test_update_service_service_id_not_found( assert mock_get2.called def test_update_service_backend_response_is_an_error_status( - self, flask_app, api100, multi_backend_connection, config, + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, service_metadata_wmts_foo ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" @@ -676,20 +690,20 @@ def test_update_service_backend_response_is_an_error_status( backend1 + "/services/wmts-foo", status_code=500, ) - abe_implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) new_process_graph = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): with pytest.raises(OpenEoApiError) as e: - abe_implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=new_process_graph) + implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=new_process_graph) assert e.value.http_status_code == 500 assert mock_patch.called - class TestInternalCollectionMetadata: def test_get_set_backends_for_collection(self): From f1904f1449a3546918ee7a121da8705fffaabbd5 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 14 Nov 2022 15:15:22 +0100 Subject: [PATCH 22/26] Issue #78, code review, Removed left-over of test coverage for API version 0.4 --- tests/conftest.py | 9 --------- tests/test_backend.py | 13 +------------ tests/test_views.py | 14 +------------- 3 files changed, 2 insertions(+), 34 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 39c7dd63..424c52c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,15 +33,6 @@ def backend2(requests_mock) -> str: return domain -# as "lib_requests_mock": make a distinction with the pytest fixture that has the same name -def set_backend_to_api_version(requests_mock, domain: str, api_version: str) -> str: - """Helper function to make the backend connection use the expected API version.""" - - # TODO: would like a nicer solution to make the backend fixtures match the expected API version. - # Good enough for now tough, just have to remember to call it in your test. - return requests_mock.get(f"{domain}/", json={"api_version": api_version}) - - @pytest.fixture def main_test_oidc_issuer() -> str: """ diff --git a/tests/test_backend.py b/tests/test_backend.py index 568988de..1f913a2e 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -15,7 +15,7 @@ from openeo_driver.users.auth import HttpAuthHandler from openeo_driver.errors import ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException from openeo.rest import OpenEoApiError, OpenEoRestError -from .conftest import DEFAULT_MEMOIZER_CONFIG, set_backend_to_api_version +from .conftest import DEFAULT_MEMOIZER_CONFIG TEST_USER = "Mr.Test" @@ -393,7 +393,6 @@ def test_create_service_succeeds( """When it gets a correct params for a new service, it successfully creates it.""" # Set up responses for creating the service in backend 1 - set_backend_to_api_version(requests_mock, backend1, "1.0.0") expected_openeo_id = "wmts-foo" location_backend_1 = backend1 + "/services/" + expected_openeo_id process_graph = {"foo": {"process_id": "foo", "arguments": {}}} @@ -429,8 +428,6 @@ def test_create_service_backend_raises_openeoapiexception( # Set up responses for creating the service in backend 1: # This time the backend raises an error, one that will be reported as a OpenEOApiException. - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", @@ -464,7 +461,6 @@ def test_create_service_backend_reraises( # Set up responses for creating the service in backend 1 # This time the backend raises an error, one that will simply be re-raised/passed on as it is. - set_backend_to_api_version(requests_mock, backend1, "1.0.0") process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", @@ -593,9 +589,6 @@ def test_update_service_succeeds( ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") - # Also test that it can skip backends that don't have the service mock_get1 = requests_mock.get( backend1 + "/services/wmts-foo", @@ -637,8 +630,6 @@ def test_update_service_service_id_not_found( ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") mock_get1 = requests_mock.get( backend1 + "/services/wmts-foo", status_code=404 @@ -679,8 +670,6 @@ def test_update_service_backend_response_is_an_error_status( ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") requests_mock.get( backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json(), diff --git a/tests/test_views.py b/tests/test_views.py index dc0d6624..a1368a9f 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -16,7 +16,7 @@ from openeo_driver.backend import ServiceMetadata from openeo_driver.testing import ApiTester, TEST_USER_AUTH_HEADER, TEST_USER, TEST_USER_BEARER_TOKEN, DictSubSet, \ RegexMatcher -from .conftest import assert_dict_subset, get_api100, get_flask_app, set_backend_to_api_version +from .conftest import assert_dict_subset, get_api100, get_flask_app class TestGeneral: @@ -1558,8 +1558,6 @@ def test_remove_service_succeeds( ): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Also test that it skips backends that don't have the service2 mock_get1 = requests_mock.get( @@ -1588,8 +1586,6 @@ def test_remove_service_service_id_not_found( ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Neither backend has the service available, and the aggregator should detect this. mock_get1 = requests_mock.get( @@ -1620,8 +1616,6 @@ def test_remove_service_backend_response_is_an_error_status( ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Will find it on the first backend, and it should skip the second backend so we don't add it to backend2. requests_mock.get( @@ -1652,8 +1646,6 @@ def test_update_service_service_succeeds( ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Also test that it skips backends that don't have the service. mock_get1 = requests_mock.get( @@ -1689,8 +1681,6 @@ def test_update_service_service_id_not_found( ): """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") # Neither backend has the service available, and the aggregator should detect this. mock_get1 = requests_mock.get( @@ -1734,8 +1724,6 @@ def test_update_service_backend_response_is_an_error_status( ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - set_backend_to_api_version(requests_mock, backend1, "1.0.0") - set_backend_to_api_version(requests_mock, backend2, "1.0.0") requests_mock.get( backend1 + "/services/wmts-foo", From 6a9c621767c111f57be267d79a66b87262cb2b32 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 14 Nov 2022 18:30:58 +0100 Subject: [PATCH 23/26] Issue #78, code review, prepend service_id on backend with the backend_id --- src/openeo_aggregator/backend.py | 111 ++++++-------- tests/test_backend.py | 250 +++++++++++++++--------------- tests/test_views.py | 253 ++++++++++++------------------- 3 files changed, 267 insertions(+), 347 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index f9283a81..ab0db071 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -656,7 +656,11 @@ def __init__( def _get_connection_and_backend_service_id( self, aggregator_service_id: str - ) -> Tuple[Union[BackendConnection, BackendConnection], str]: + ) -> Tuple[BackendConnection, str]: + """Get connection to the backend and the corresponding service ID in that backend. + + raises: ServiceNotFoundException when service_id does not exist in any of the backends. + """ backend_service_id, backend_id = ServiceIdMapping.parse_aggregator_service_id( backends=self._backends, aggregator_service_id=aggregator_service_id @@ -718,28 +722,23 @@ def merge(services, to_add): def service_info(self, user_id: str, service_id: str) -> ServiceMetadata: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-service""" - # con, backend_service_id = self._get_connection_and_backend_service_id(service_id) - # with con.authenticated_from_request(request=flask.request, user=User(user_id)): - # try: - # service_json = con.get(f"/services/{backend_service_id}").json() - # except Exception as e: - # _log.debug(f"Failed to get service with ID={backend_service_id} from backend with ID={con.id}: {e!r}", exc_info=True) - # raise - # else: - # service_json["id"] = ServiceIdMapping.get_aggregator_service_id(service_json["id"], con.id) - # return ServiceMetadata.from_dict(service_json) - - for con in self._backends: - with con.authenticated_from_request(request=flask.request, user=User(user_id)): - try: - service_json = con.get(f"/services/{service_id}").json() - except Exception as e: - _log.debug(f"No service with ID={service_id} in backend with ID={con.id}: {e!r}", exc_info=True) - continue - else: - return ServiceMetadata.from_dict(service_json) - - raise ServiceNotFoundException(service_id) + con, backend_service_id = self._get_connection_and_backend_service_id(service_id) + with con.authenticated_from_request(request=flask.request, user=User(user_id)): + try: + service_json = con.get(f"/services/{backend_service_id}").json() + except (OpenEoApiError) as e: + if e.http_status_code == 404: + # Expected error + _log.debug(f"No service with ID={service_id!r} in backend with ID={con.id!r}: {e!r}", exc_info=True) + raise ServiceNotFoundException(service_id=service_id) from e + raise + except Exception as e: + _log.debug(f"Failed to get service with ID={backend_service_id} from backend with ID={con.id}: {e!r}", exc_info=True) + raise + else: + # Adapt the service ID so it points to the aggregator, with the backend ID included. + service_json["id"] = ServiceIdMapping.get_aggregator_service_id(service_json["id"], con.id) + return ServiceMetadata.from_dict(service_json) def create_service(self, user_id: str, process_graph: dict, service_type: str, api_version: str, configuration: dict) -> str: @@ -768,72 +767,50 @@ def create_service(self, user_id: str, process_graph: dict, service_type: str, a except (OpenEoRestError, OpenEoClientException) as e: raise OpenEOApiException(f"Failed to create secondary service on backend {backend_id!r}: {e!r}") - return service.service_id - - def _find_connection_with_service_id(self, user_id: str, service_id: str) -> Optional[BackendConnection]: - """Get connection for the backend that contains the service, return None if not found.""" - - # Search all services on the backends. - for con in self._backends: - with con.authenticated_from_request(request=flask.request, user=User(user_id)): - try: - _ = con.get(f"/services/{service_id}") - except OpenEoApiError as e: - if e.http_status_code == 404: - # Expected error - _log.debug(f"No service with ID={service_id!r} in backend with ID={con.id!r}: {e!r}", exc_info=True) - continue - else: - _log.warning(f"Failed to get service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) - raise e - except Exception as e: - _log.warning(f"Failed to get service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) - raise e - else: - return con - return None + return ServiceIdMapping.get_aggregator_service_id(service.service_id, backend_id) def remove_service(self, user_id: str, service_id: str) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/delete-service""" - con = self._find_connection_with_service_id(user_id, service_id) - if not con: - raise ServiceNotFoundException(service_id) + # Will raise ServiceNotFoundException if service_id does not exist in any of the backends. + con, backend_service_id = self._get_connection_and_backend_service_id(service_id) with con.authenticated_from_request(request=flask.request, user=User(user_id)): try: - con.delete(f"/services/{service_id}", expected_status=204) - except (OpenEoApiError, OpenEOApiException) as e: - # TODO: maybe we should just let these exception straight go to the caller without logging it here. - # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. - _log.warning(f"Failed to delete service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + con.delete(f"/services/{backend_service_id}", expected_status=204) + except (OpenEoApiError) as e: + if e.http_status_code == 404: + # Expected error + _log.debug(f"No service with ID={service_id!r} in backend with ID={con.id!r}: {e!r}", exc_info=True) + raise ServiceNotFoundException(service_id=service_id) from e + _log.warning(f"Failed to delete service {backend_service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise except Exception as e: - _log.warning(f"Failed to delete service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + _log.warning(f"Failed to delete service {backend_service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise OpenEOApiException( - f"Failed to delete service {service_id!r} on backend {con.id!r}: {e!r}" + f"Failed to delete service {backend_service_id!r} on backend {con.id!r}: {e!r}" ) from e def update_service(self, user_id: str, service_id: str, process_graph: dict) -> None: """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/update-service""" - con = self._find_connection_with_service_id(user_id, service_id) - if not con: - raise ServiceNotFoundException(service_id) + # Will raise ServiceNotFoundException if service_id does not exist in any of the backends. + con, backend_service_id = self._get_connection_and_backend_service_id(service_id) with con.authenticated_from_request(request=flask.request, user=User(user_id)): try: json = {"process": {"process_graph": process_graph}} - con.patch(f"/services/{service_id}", json=json, expected_status=204) - except (OpenEoApiError, OpenEOApiException) as e: - # TODO: maybe we should just let these exception straight go to the caller without logging it here. - # Logging it here seems prudent and more consistent with the handling of unexpected exceptions below. - _log.warning(f"Failed to update service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + con.patch(f"/services/{backend_service_id}", json=json, expected_status=204) + except (OpenEoApiError) as e: + if e.http_status_code == 404: + # Expected error + _log.debug(f"No service with ID={backend_service_id!r} in backend with ID={con.id!r}: {e!r}", exc_info=True) + raise ServiceNotFoundException(service_id=service_id) from e raise except Exception as e: - _log.warning(f"Failed to update service {service_id!r} from {con.id!r}: {e!r}", exc_info=True) + _log.warning(f"Failed to update service {backend_service_id!r} from {con.id!r}: {e!r}", exc_info=True) raise OpenEOApiException( - f"Failed to update service {service_id!r} from {con.id!r}: {e!r}" + f"Failed to update service {backend_service_id!r} from {con.id!r}: {e!r}" ) from e diff --git a/tests/test_backend.py b/tests/test_backend.py index 1f913a2e..13de88bf 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -354,8 +354,10 @@ def test_service_info_succeeds( service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" - requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) + json_wmts_foo = service_metadata_wmts_foo.prepare_for_json() + json_wms_bar = service_metadata_wms_bar.prepare_for_json() + requests_mock.get(backend1 + "/services/wmts-foo", json=json_wmts_foo) + requests_mock.get(backend2 + "/services/wms-bar", json=json_wms_bar) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) @@ -363,21 +365,30 @@ def test_service_info_succeeds( # Check the expected metadata on *both* of the services. with flask_app.test_request_context(headers=headers): - actual_service1 = implementation.service_info(user_id=TEST_USER, service_id="wmts-foo") - assert actual_service1 == service_metadata_wmts_foo + actual_service1 = implementation.service_info(user_id=TEST_USER, service_id="b1-wmts-foo") + + json = dict(json_wmts_foo) + json["id"] = "b1-" + json["id"] + expected_service1 = ServiceMetadata.from_dict(json) + + assert actual_service1 == expected_service1 with flask_app.test_request_context(headers=headers): - actual_service2 = implementation.service_info(user_id=TEST_USER, service_id="wms-bar") - assert actual_service2 == service_metadata_wms_bar + actual_service2 = implementation.service_info(user_id=TEST_USER, service_id="b2-wms-bar") - def test_service_info_wrong_id( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, - service_metadata_wmts_foo, service_metadata_wms_bar + json = dict(json_wms_bar) + json["id"] = "b2-" + json["id"] + expected_service2 = ServiceMetadata.from_dict(json) + + assert actual_service2 == expected_service2 + + def test_service_info_wrong_backend_id( + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock, + service_metadata_wmts_foo ): """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) @@ -385,7 +396,24 @@ def test_service_info_wrong_id( with flask_app.test_request_context(headers=headers): with pytest.raises(ServiceNotFoundException): - implementation.service_info(user_id=TEST_USER, service_id="doesnotexist") + implementation.service_info(user_id=TEST_USER, service_id="backenddoesnotexist-wtms-foo") + + def test_service_info_wrong_service_id( + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock, + ): + """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" + + requests_mock.get(backend1 + "/services/service-does-not-exist", status_code=404) + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER + + with flask_app.test_request_context(headers=headers): + with pytest.raises(ServiceNotFoundException): + implementation.service_info(user_id=TEST_USER, service_id="b1-service-does-not-exist") + + assert requests_mock.called def test_create_service_succeeds( self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock @@ -393,13 +421,16 @@ def test_create_service_succeeds( """When it gets a correct params for a new service, it successfully creates it.""" # Set up responses for creating the service in backend 1 - expected_openeo_id = "wmts-foo" - location_backend_1 = backend1 + "/services/" + expected_openeo_id + backend_service_id = "wmts-foo" + # The aggregator should prepend the service_id with the backend_id + expected_service_id = "b1-wmts-foo" + + location_backend_1 = backend1 + "/services/" + backend_service_id process_graph = {"foo": {"process_id": "foo", "arguments": {}}} requests_mock.post( backend1 + "/services", headers={ - "OpenEO-Identifier": expected_openeo_id, + "OpenEO-Identifier": backend_service_id, "Location": location_backend_1 }, status_code=201 @@ -417,12 +448,12 @@ def test_create_service_succeeds( api_version="1.0.0", configuration={} ) - assert actual_openeo_id == expected_openeo_id + assert actual_openeo_id == expected_service_id @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) def test_create_service_backend_raises_openeoapiexception( self, flask_app, api100, multi_backend_connection, config, catalog, - backend1, backend2, requests_mock, exception_class + backend1, requests_mock, exception_class ): """When the backend raises a general exception the aggregator raises an OpenEOApiException.""" @@ -483,91 +514,66 @@ def test_create_service_backend_reraises( ) def test_remove_service_succeeds( - self, flask_app, api100, multi_backend_connection, config, catalog, - backend1, backend2, requests_mock, service_metadata_wmts_foo + self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock ): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" - # Also test that it can skip backends that don't have the service - mock_get1 = requests_mock.get( - backend1 + "/services/wmts-foo", - status_code=404 - ) - # Delete should succeed in backend2 so service should be present first. - mock_get2 = requests_mock.get( - backend2 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 - ) - mock_delete = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) + mock_delete = requests_mock.delete(backend1 + "/services/wmts-foo", status_code=204) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): - implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + implementation.remove_service(user_id=TEST_USER, service_id="b1-wmts-foo") # Make sure the aggregator asked the backend to remove the service. assert mock_delete.called - # Check the other mocks were called too, just to be sure. - assert mock_get1.called - assert mock_get2.called + def test_remove_service_but_backend_id_not_found( + self, flask_app, api100, multi_backend_connection, config, catalog, + ): + """When the backend ID/prefix does not exist then the aggregator raises an ServiceNotFoundException.""" + + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER - def test_remove_service_service_id_not_found( + # Case 1: the backend doesn't even exist + with flask_app.test_request_context(headers=headers): + with pytest.raises(ServiceNotFoundException): + implementation.remove_service(user_id=TEST_USER, service_id="doesnotexist-wmts-foo") + + def test_remove_service_but_service_id_not_found( self, flask_app, api100, multi_backend_connection, config, catalog, - backend1, backend2, requests_mock, service_metadata_wmts_foo + backend1, requests_mock ): - """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" + """When the service ID does not exist for the specified backend then the aggregator raises an ServiceNotFoundException.""" - # Neither backend has the service available, and the aggregator should detect this. - mock_get1 = requests_mock.get( - backend1 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=404 - ) - mock_get2 = requests_mock.get( - backend2 + "/services/wmts-foo", - status_code=404 - ) - - # These requests should not be executed, so check they are not called. - mock_delete1 = requests_mock.delete( - backend2 + "/services/wmts-foo", - status_code=204 - ) - mock_delete2 = requests_mock.delete( - backend2 + "/services/wmts-foo", - status_code=204 - ) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) headers = TEST_USER_AUTH_HEADER + # The backend exists but the service ID does not. + mock_delete1 = requests_mock.delete( + backend1 + "/services/doesnotexist", + status_code=404 + ) with flask_app.test_request_context(headers=headers): with pytest.raises(ServiceNotFoundException): - implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + implementation.remove_service(user_id=TEST_USER, service_id="b1-doesnotexist") - assert not mock_delete1.called - assert not mock_delete2.called - # Check the other mocks were called too, just to be sure. - assert mock_get1.called - assert mock_get2.called + # This should have tried to delete it on the backend so the mock must be called. + assert mock_delete1.called def test_remove_service_backend_response_is_an_error_status( self, flask_app, api100, multi_backend_connection, config, catalog, - backend1, requests_mock, service_metadata_wmts_foo - ): - """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" + backend1, requests_mock + ): + """When the backend response is an HTTP error status then the aggregator raises an OpenEoApiError.""" - # Will find it on the first backend, and it should skip the second backend so we don't add it to backend2. - requests_mock.get( - backend1 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 - ) requests_mock.delete(backend1 + "/services/wmts-foo", status_code=500) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) @@ -576,32 +582,20 @@ def test_remove_service_backend_response_is_an_error_status( with flask_app.test_request_context(headers=headers): with pytest.raises(OpenEoApiError) as e: - implementation.remove_service(user_id=TEST_USER, service_id="wmts-foo") + implementation.remove_service(user_id=TEST_USER, service_id="b1-wmts-foo") # If the backend reports HTTP 400/500, we would expect the same status code from the aggregator. # TODO: Statement above is an assumption. Is that really what we expect? assert e.value.http_status_code == 500 - # TODO: this test still fails with API version 1.0.0 def test_update_service_succeeds( self, flask_app, api100, multi_backend_connection, config, catalog, - backend1, backend2, requests_mock, service_metadata_wmts_foo + backend1, requests_mock ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" - # Also test that it can skip backends that don't have the service - mock_get1 = requests_mock.get( - backend1 + "/services/wmts-foo", - status_code=404 - ) - # Update should succeed in backend2 so service should be present - mock_get2 = requests_mock.get( - backend2 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 - ) mock_patch = requests_mock.patch( - backend2 + "/services/wmts-foo", + backend1 + "/services/wmts-foo", status_code=204, ) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) @@ -611,7 +605,7 @@ def test_update_service_succeeds( headers = TEST_USER_AUTH_HEADER with flask_app.test_request_context(headers=headers): - implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + implementation.update_service(user_id=TEST_USER, service_id="b1-wmts-foo", process_graph=process_graph_after) # Make sure the aggregator asked the backend to remove the service. assert mock_patch.called @@ -620,33 +614,31 @@ def test_update_service_succeeds( expected_process = {"process": {"process_graph": process_graph_after}} assert mock_patch.last_request.json() == expected_process - # Check the other mocks were called too, just to be sure. - assert mock_get1.called - assert mock_get2.called + def test_update_service_but_backend_id_does_not_exist( + self, flask_app, api100, multi_backend_connection, config, catalog, + ): + """When the backend ID/prefix does not exist then the aggregator raises an ServiceNotFoundException.""" - def test_update_service_service_id_not_found( + processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) + implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) + process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + headers = TEST_USER_AUTH_HEADER + + with flask_app.test_request_context(headers=headers): + with pytest.raises(ServiceNotFoundException): + implementation.update_service(user_id=TEST_USER, service_id="doesnotexist-wmts-foo", process_graph=process_graph_after) + + def test_update_service_but_service_id_not_found( self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock ): - """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" + """When the service ID does not exist for the specified backend then the aggregator raises an ServiceNotFoundException.""" - mock_get1 = requests_mock.get( - backend1 + "/services/wmts-foo", - status_code=404 - ) - mock_get2 = requests_mock.get( - backend2 + "/services/wmts-foo", - status_code=404 - ) - # These requests should not be executed, so check they are not called. mock_patch1 = requests_mock.patch( - backend1 + "/services/wmts-foo", - status_code=204, - ) - mock_patch2 = requests_mock.patch( - backend2 + "/services/wmts-foo", - status_code=204, + backend1 + "/services/doesnotexist", + status_code=404, ) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) @@ -656,25 +648,16 @@ def test_update_service_service_id_not_found( with flask_app.test_request_context(headers=headers): with pytest.raises(ServiceNotFoundException): - implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=process_graph_after) + implementation.update_service(user_id=TEST_USER, service_id="b1-doesnotexist", process_graph=process_graph_after) - assert not mock_patch1.called - assert not mock_patch2.called - # Check the other mocks were called too, just to be sure. - assert mock_get1.called - assert mock_get2.called + assert mock_patch1.called def test_update_service_backend_response_is_an_error_status( self, flask_app, api100, multi_backend_connection, config, catalog, - backend1, backend2, requests_mock, service_metadata_wmts_foo + backend1, requests_mock ): - """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" + """When the backend response is an HTTP error status then the aggregator raises an OpenEoApiError.""" - requests_mock.get( - backend1 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 - ) mock_patch = requests_mock.patch( backend1 + "/services/wmts-foo", status_code=500, @@ -687,7 +670,7 @@ def test_update_service_backend_response_is_an_error_status( with flask_app.test_request_context(headers=headers): with pytest.raises(OpenEoApiError) as e: - implementation.update_service(user_id=TEST_USER, service_id="wmts-foo", process_graph=new_process_graph) + implementation.update_service(user_id=TEST_USER, service_id="b1-wmts-foo", process_graph=new_process_graph) assert e.value.http_status_code == 500 assert mock_patch.called @@ -1674,6 +1657,29 @@ def test_parse_aggregator_job_id_fail(self, multi_backend_connection): ) +from openeo_aggregator.backend import ServiceIdMapping +class TestServiceIdMapping: + + def test_get_aggregator_job_id(self): + assert ServiceIdMapping.get_aggregator_service_id( + backend_service_id="service-x17-abc", backend_id="vito" + ) == "vito-service-x17-abc" + + def test_parse_aggregator_job_id(self, multi_backend_connection): + assert ServiceIdMapping.parse_aggregator_service_id( + backends=multi_backend_connection, aggregator_service_id="b1-serv021b" + ) == ("serv021b", "b1") + assert ServiceIdMapping.parse_aggregator_service_id( + backends=multi_backend_connection, aggregator_service_id="b2-someservice-321-ab14jh" + ) == ("someservice-321-ab14jh", "b2") + + def test_parse_aggregator_job_id_fail(self, multi_backend_connection): + with pytest.raises(ServiceNotFoundException): + ServiceIdMapping.parse_aggregator_service_id( + backends=multi_backend_connection, aggregator_service_id="b3-b6tch-j0b-o123423" + ) + + class TestAggregatorProcessing: def test_get_process_registry( self, diff --git a/tests/test_views.py b/tests/test_views.py index a1368a9f..f3c680e1 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1335,19 +1335,6 @@ def service_metadata_wmts_foo(self): # not setting "created": This is used to test creating a service. ) - @pytest.fixture - def service_metadata_wms_bar(self): - return ServiceMetadata( - id="wms-bar", - process={"process_graph": {"bar": {"process_id": "bar", "arguments": {}}}}, - url='https://oeo.net/wms/bar', - type="WMS", - enabled=True, - configuration={"version": "0.5.8"}, - attributes={}, - title="Test WMS service" - # not setting "created": This is used to test creating a service. - ) def test_service_types_simple(self, api100, backend1, backend2, requests_mock): """Given 2 backends but only 1 backend has a single service, then the aggregator @@ -1422,64 +1409,56 @@ def test_service_types_merging(self, api100, backend1, backend2, requests_mock): assert actual_service_types == expected_service_types def test_service_info( - self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar + self, api100, backend1, requests_mock ): - """When it gets a correct service ID, it returns the expected ServiceMetadata.""" - - requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) + """When it gets a correct service ID, it returns the expected service's metadata as JSON.""" + + json_wmts_foo = { + "id": "wmts-foo", + "process": {"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + "url": "https://oeo.net/wmts/foo", + "type": "WMTS", + "enabled": "True", + "configuration": {"version": "0.5.8"}, + "attributes": {}, + "title": "Test WMTS service" + } + requests_mock.get(backend1 + "/services/wmts-foo", json=json_wmts_foo) api100.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) - # Retrieve and verify the metadata for both services - resp = api100.get("/services/wmts-foo").assert_status_code(200) - actual_service1 = ServiceMetadata( - id=resp.json["id"], - process=resp.json["process"], - url=resp.json["url"], - type=resp.json["type"], - enabled=resp.json["enabled"], - configuration=resp.json["configuration"], - attributes=resp.json["attributes"], - title=resp.json["title"], - ) - assert actual_service1 == service_metadata_wmts_foo - - resp = api100.get("/services/wms-bar").assert_status_code(200) - actual_service2 = ServiceMetadata( - id=resp.json["id"], - process=resp.json["process"], - url=resp.json["url"], - type=resp.json["type"], - enabled=resp.json["enabled"], - configuration=resp.json["configuration"], - attributes=resp.json["attributes"], - title=resp.json["title"], - ) - assert actual_service2 == service_metadata_wms_bar + resp = api100.get("/services/b1-wmts-foo").assert_status_code(200) - def test_service_info_wrong_id( - self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar - ): - """When it gets a non-existent service ID, it returns HTTP Status 404, Not found.""" + expected_json_wmts_foo = dict(json_wmts_foo) + expected_json_wmts_foo["id"] = "b1-" + json_wmts_foo["id"] + assert resp.json == expected_json_wmts_foo - requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) - requests_mock.get(backend2 + "/services/wms-bar", json=service_metadata_wms_bar.prepare_for_json()) + def test_service_info_wrong_id(self, api100): + """When it gets a non-existent service ID, the aggregator responds with HTTP 404, not found.""" api100.set_auth_bearer_token(token=TEST_USER_BEARER_TOKEN) - api100.get("/services/doesnotexist").assert_status_code(404) + # The backend ID is wrong. + api100.get("/services/doesnotexist-someservice").assert_status_code(404) + + # The backend ID exists but the service ID is wrong. + api100.get("/services/b1-doesnotexist").assert_status_code(404) def test_create_wmts(self, api100, requests_mock, backend1): + """When the payload is correct the service should be successfully created, + the service ID should be prepended with the backend ID, + and location should point to the aggregator, not to the backend directly. + """ api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - expected_openeo_id = 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' - # The aggregator MUST NOT point to the actual instance but to its own endpoint. + backend_service_id = 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' + expected_openeo_id = "b1-" + backend_service_id + + # The aggregator MUST NOT point to the backend instance but to its own endpoint. # This is handled by the openeo python driver in openeo_driver.views.services_post. expected_location = "/openeo/1.0.0/services/" + expected_openeo_id # However, backend1 must report its OWN location. - location_backend_1 = backend1 + "/services" + expected_openeo_id + location_backend_1 = backend1 + "/services" + backend_service_id process_graph = {"foo": {"process_id": "foo", "arguments": {}}} - # The process_graph/process format is slightly different between api v0.4 and v1.0 post_data = { "type": 'WMTS', "process": { @@ -1492,7 +1471,7 @@ def test_create_wmts(self, api100, requests_mock, backend1): requests_mock.post( backend1 + "/services", headers={ - "OpenEO-Identifier": expected_openeo_id, + "OpenEO-Identifier": backend_service_id, "Location": location_backend_1 }, status_code=201 @@ -1500,16 +1479,17 @@ def test_create_wmts(self, api100, requests_mock, backend1): resp = api100.post('/services', json=post_data).assert_status_code(201) - assert resp.headers['OpenEO-Identifier'] == 'c63d6c27-c4c2-4160-b7bd-9e32f582daec' + assert resp.headers['OpenEO-Identifier'] == expected_openeo_id assert resp.headers['Location'] == expected_location # ProcessGraphMissingException and ProcessGraphInvalidException are well known reasons for a bad client request. @pytest.mark.parametrize("exception_class", [ProcessGraphMissingException, ProcessGraphInvalidException]) def test_create_wmts_reports_400_client_error(self, api100, requests_mock, backend1, exception_class): - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + """When the backend raises exceptions that are typically a bad request / HTTP 400, then + we expect the aggregator to return a HTTP 400 status code.""" + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) process_graph = {"foo": {"process_id": "foo", "arguments": {}}} - # The process_graph/process format is slightly different between api v0.4 and v1.0 post_data = { "type": 'WMTS', "process": { @@ -1532,10 +1512,12 @@ def test_create_wmts_reports_400_client_error(self, api100, requests_mock, backe # OpenEoApiError, OpenEoRestError: more general errors we can expect to lead to a HTTP 500 server error. @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) def test_create_wmts_reports_500_server_error(self, api100, requests_mock, backend1, exception_class): + """When the backend raises exceptions that are typically a server error / HTTP 500, then + we expect the aggregator to return a HTTP 500 status code.""" + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) process_graph = {"foo": {"process_id": "foo", "arguments": {}}} - # The process_graph/process format is slightly different between api v0.4 and v1.0 post_data = { "type": 'WMTS', "process": { @@ -1554,70 +1536,51 @@ def test_create_wmts_reports_500_server_error(self, api100, requests_mock, backe assert resp.status_code == 500 def test_remove_service_succeeds( - self, api100, requests_mock, backend1, backend2, service_metadata_wmts_foo + self, api100, requests_mock, backend1 ): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - # Also test that it skips backends that don't have the service2 - mock_get1 = requests_mock.get( - backend1 + "/services/wmts-foo", - status_code=404 - ) - # Delete should succeed in backend2 so service should be present first. - mock_get2 = requests_mock.get( - backend2 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 - ) - mock_delete = requests_mock.delete(backend2 + "/services/wmts-foo", status_code=204) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + mock_delete = requests_mock.delete(backend1 + "/services/wmts-foo", status_code=204) - resp = api100.delete("/services/wmts-foo") + resp = api100.delete("/services/b1-wmts-foo") assert resp.status_code == 204 # Make sure the aggregator asked the backend to remove the service. assert mock_delete.called - # Verify the aggregator did query the backends to find the service. - assert mock_get1.called - assert mock_get1.called - def test_remove_service_service_id_not_found( - self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo - ): - """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" + + def test_remove_service_but_backend_id_not_found(self, api100): + """When the service ID does not exist then the aggregator responds with HTTP 404, not found.""" + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - # Neither backend has the service available, and the aggregator should detect this. - mock_get1 = requests_mock.get( + resp = api100.delete("/services/wmts-foo") + + assert resp.status_code == 404 + + def test_remove_service_but_service_id_not_found( + self, api100, backend1, requests_mock + ): + """When the service ID does not exist then the aggregator responds with HTTP 404, not found.""" + + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + mock_delete = requests_mock.delete( backend1 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=404 - ) - mock_get2 = requests_mock.get( - backend2 + "/services/wmts-foo", status_code=404, - ) - mock_delete = requests_mock.delete( - backend2 + "/services/wmts-foo", - status_code=204, # deliberately avoid 404 so we know 404 comes from aggregator. ) - resp = api100.delete("/services/wmts-foo") + resp = api100.delete("/services/b1-wmts-foo") assert resp.status_code == 404 - # Verify the aggregator did not call the backend to remove the service. - assert not mock_delete.called - # Verify the aggregator did query the backends to find the service. - assert mock_get1.called - assert mock_get2.called + assert mock_delete.called def test_remove_service_backend_response_is_an_error_status( - self, api100, requests_mock, backend1, backend2, service_metadata_wmts_foo + self, api100, requests_mock, backend1, service_metadata_wmts_foo ): - """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + """When the backend response is an error, HTTP 500, then the aggregator also responds with HTTP 500 status.""" - # Will find it on the first backend, and it should skip the second backend so we don't add it to backend2. + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) requests_mock.get( backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json(), @@ -1634,7 +1597,7 @@ def test_remove_service_backend_response_is_an_error_status( } ) - resp = api100.delete("/services/wmts-foo") + resp = api100.delete("/services/b1-wmts-foo") assert resp.status_code == 500 # Verify the aggregator effectively asked the backend to remove the service, @@ -1642,94 +1605,68 @@ def test_remove_service_backend_response_is_an_error_status( assert mock_delete.called def test_update_service_service_succeeds( - self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo + self, api100, backend1, requests_mock, service_metadata_wmts_foo ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - # Also test that it skips backends that don't have the service. - mock_get1 = requests_mock.get( - backend1 + "/services/wmts-foo", - status_code=404 - ) - mock_get2 = requests_mock.get( - backend2 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 - ) mock_patch = requests_mock.patch( - backend2 + "/services/wmts-foo", + backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json(), status_code=204 ) process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} json_payload = {"process": {"process_graph": process_graph}} - resp = api100.patch("/services/wmts-foo", json=json_payload) + resp = api100.patch("/services/b1-wmts-foo", json=json_payload) assert resp.status_code == 204 # Make sure the aggregator asked the backend to update the service. assert mock_patch.called assert mock_patch.last_request.json() == json_payload - - # Check other mocks were called, to be sure it searched before updating. - assert mock_get1.called - assert mock_get2.called - def test_update_service_service_id_not_found( - self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo + + def test_update_service_but_backend_id_not_found( + self, api100 ): - """When the service ID does not exist then the aggregator raises an ServiceNotFoundException.""" + """When the service ID does not exist because the backend prefix is wrong, then the aggregator responds with HTTP 404, not found.""" + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} + json_payload = {"process": {"process_graph": process_graph}} - # Neither backend has the service available, and the aggregator should detect this. - mock_get1 = requests_mock.get( - backend1 + "/services/wmts-foo", + resp = api100.patch("/services/backenddoesnotexist-someservice", json=json_payload) + + assert resp.status_code == 404 + + def test_update_service_service_id_not_found( + self, api100, backend1, requests_mock, service_metadata_wmts_foo + ): + """When the service ID does not exist for the specified backend, then the aggregator responds with HTTP 404, not found.""" + + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) + mock_patch = requests_mock.patch( + backend1 + "/services/servicedoesnotexist", json=service_metadata_wmts_foo.prepare_for_json(), status_code=404 ) - mock_get2 = requests_mock.get( - backend2 + "/services/wmts-foo", - status_code=404, - ) - # The aggregator should not execute a HTTP patch, so we check that it does not call these two. - mock_patch1 = requests_mock.patch( - backend1 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=204 - ) - mock_patch2 = requests_mock.patch( - backend2 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=204 # deliberately avoid 404 so we know 404 comes from aggregator. - ) process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} json_payload = {"process": {"process_graph": process_graph}} - resp = api100.patch("/services/wmts-foo", json=json_payload) + resp = api100.patch("/services/b1-servicedoesnotexist", json=json_payload) assert resp.status_code == 404 - # Verify that the aggregator did not try to call patch on the backend. - assert not mock_patch1.called - assert not mock_patch2.called - # Verify that the aggregator asked the backend to remove the service. - assert mock_get1.called - assert mock_get2.called + assert mock_patch.called # TODO: for now, not bothering with HTTP 400 in the backend. To be decided if this is necessary. - # @pytest.mark.parametrize("backend_http_status", [400, 500]) @pytest.mark.parametrize("backend_http_status", [500]) def test_update_service_backend_response_is_an_error_status( - self, api100, backend1, backend2, requests_mock, service_metadata_wmts_foo, backend_http_status + self, api100, backend1, requests_mock, backend_http_status ): """When the backend response is an error HTTP 400/500 then the aggregator raises an OpenEoApiError.""" - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - requests_mock.get( - backend1 + "/services/wmts-foo", - json=service_metadata_wmts_foo.prepare_for_json(), - status_code=200 - ) + api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) mock_patch = requests_mock.patch( backend1 + "/services/wmts-foo", status_code=backend_http_status, @@ -1743,7 +1680,7 @@ def test_update_service_backend_response_is_an_error_status( process_graph = {"bar": {"process_id": "bar", "arguments": {"new_arg": "somevalue"}}} json_payload = {"process": {"process_graph": process_graph}} - resp = api100.patch("/services/wmts-foo", json=json_payload) + resp = api100.patch("/services/b1-wmts-foo", json=json_payload) assert resp.status_code == backend_http_status assert mock_patch.called From 7cfd11b1d83fc295936d70b1bc8fb21f856613ed Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Mon, 14 Nov 2022 21:29:06 +0100 Subject: [PATCH 24/26] Issue #78, clean up authentication code in test_backend.py --- tests/test_backend.py | 71 +++++++++++-------------------------------- 1 file changed, 18 insertions(+), 53 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 13de88bf..048ae107 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -245,10 +245,8 @@ def test_list_services_simple( requests_mock.get(backend2 + "/services", json=services2) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): actual_services = implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in services1 @@ -273,10 +271,7 @@ def test_list_services_merged( processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): actual_services = implementation.list_services(user_id=TEST_USER) expected_services = [service_metadata_wmts_foo, service_metadata_wms_bar] @@ -332,10 +327,8 @@ def test_list_services_merged_multiple( requests_mock.get(backend2 + "/services", json=services2) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): actual_services = implementation.list_services(user_id=TEST_USER) # Construct expected result. We have get just data from the service in @@ -360,11 +353,9 @@ def test_service_info_succeeds( requests_mock.get(backend2 + "/services/wms-bar", json=json_wms_bar) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER # Check the expected metadata on *both* of the services. - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): actual_service1 = implementation.service_info(user_id=TEST_USER, service_id="b1-wmts-foo") json = dict(json_wmts_foo) @@ -373,7 +364,7 @@ def test_service_info_succeeds( assert actual_service1 == expected_service1 - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): actual_service2 = implementation.service_info(user_id=TEST_USER, service_id="b2-wms-bar") json = dict(json_wms_bar) @@ -391,10 +382,8 @@ def test_service_info_wrong_backend_id( requests_mock.get(backend1 + "/services/wmts-foo", json=service_metadata_wmts_foo.prepare_for_json()) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(ServiceNotFoundException): implementation.service_info(user_id=TEST_USER, service_id="backenddoesnotexist-wtms-foo") @@ -406,10 +395,8 @@ def test_service_info_wrong_service_id( requests_mock.get(backend1 + "/services/service-does-not-exist", status_code=404) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(ServiceNotFoundException): implementation.service_info(user_id=TEST_USER, service_id="b1-service-does-not-exist") @@ -437,10 +424,8 @@ def test_create_service_succeeds( ) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): actual_openeo_id = implementation.create_service( user_id=TEST_USER, process_graph=process_graph, @@ -466,10 +451,8 @@ def test_create_service_backend_raises_openeoapiexception( ) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(OpenEOApiException): implementation.create_service( user_id=TEST_USER, @@ -499,10 +482,8 @@ def test_create_service_backend_reraises( ) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): # These exception types should be re-raised, not become an OpenEOApiException. with pytest.raises(exception_class): implementation.create_service( @@ -521,10 +502,8 @@ def test_remove_service_succeeds( mock_delete = requests_mock.delete(backend1 + "/services/wmts-foo", status_code=204) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): implementation.remove_service(user_id=TEST_USER, service_id="b1-wmts-foo") # Make sure the aggregator asked the backend to remove the service. @@ -537,11 +516,9 @@ def test_remove_service_but_backend_id_not_found( processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER # Case 1: the backend doesn't even exist - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(ServiceNotFoundException): implementation.remove_service(user_id=TEST_USER, service_id="doesnotexist-wmts-foo") @@ -553,15 +530,13 @@ def test_remove_service_but_service_id_not_found( processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER # The backend exists but the service ID does not. mock_delete1 = requests_mock.delete( backend1 + "/services/doesnotexist", status_code=404 ) - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(ServiceNotFoundException): implementation.remove_service(user_id=TEST_USER, service_id="b1-doesnotexist") @@ -577,10 +552,8 @@ def test_remove_service_backend_response_is_an_error_status( requests_mock.delete(backend1 + "/services/wmts-foo", status_code=500) processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(OpenEoApiError) as e: implementation.remove_service(user_id=TEST_USER, service_id="b1-wmts-foo") @@ -601,10 +574,8 @@ def test_update_service_succeeds( processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): implementation.update_service(user_id=TEST_USER, service_id="b1-wmts-foo", process_graph=process_graph_after) # Make sure the aggregator asked the backend to remove the service. @@ -622,10 +593,8 @@ def test_update_service_but_backend_id_does_not_exist( processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(ServiceNotFoundException): implementation.update_service(user_id=TEST_USER, service_id="doesnotexist-wmts-foo", process_graph=process_graph_after) @@ -643,10 +612,8 @@ def test_update_service_but_service_id_not_found( processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) process_graph_after = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(ServiceNotFoundException): implementation.update_service(user_id=TEST_USER, service_id="b1-doesnotexist", process_graph=process_graph_after) @@ -665,10 +632,8 @@ def test_update_service_backend_response_is_an_error_status( processing = AggregatorProcessing(backends=multi_backend_connection, catalog=catalog, config=config) implementation = AggregatorSecondaryServices(backends=multi_backend_connection, processing=processing) new_process_graph = {"bar": {"process_id": "bar", "arguments": {"arg1": "bar"}}} - api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) - headers = TEST_USER_AUTH_HEADER - with flask_app.test_request_context(headers=headers): + with flask_app.test_request_context(headers=TEST_USER_AUTH_HEADER): with pytest.raises(OpenEoApiError) as e: implementation.update_service(user_id=TEST_USER, service_id="b1-wmts-foo", process_graph=new_process_graph) From 0d2e0264813ff4c59bd6e387a15b167aff41d54e Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Tue, 15 Nov 2022 09:59:30 +0100 Subject: [PATCH 25/26] Issue #78 Updated changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0df8a589..5a1b853d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- Implement secondary services [#78](https://github.com/Open-EO/openeo-aggregator/issues/78)) + ## [0.6.x] ### Added From 4b80f9b9457cc7556b7ef21a76e136dbefcd33f6 Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Tue, 15 Nov 2022 10:51:46 +0100 Subject: [PATCH 26/26] Issue #78 Removed unused arguments from test functions --- tests/test_backend.py | 48 ++++++++++++++++++++++--------------------- tests/test_views.py | 22 +++++++------------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/tests/test_backend.py b/tests/test_backend.py index 048ae107..fecd44fc 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -162,7 +162,9 @@ def test_service_types_simple( service_types = implementation.service_types() assert service_types == single_service_type - def test_service_types_merging(self, multi_backend_connection, config, catalog, backend1, backend2, requests_mock): + def test_service_types_merging(self, multi_backend_connection, config, catalog, + backend1, backend2, requests_mock + ): """Given 2 backends with each 1 service type, then the aggregator lists both service types.""" service_type_1 = { "WMTS": { @@ -233,8 +235,8 @@ def service_metadata_wms_bar(self): ) def test_list_services_simple( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, - service_metadata_wmts_foo + self, flask_app, multi_backend_connection, config, catalog, + backend1, backend2, requests_mock, service_metadata_wmts_foo ): """Given 2 backends but only 1 backend has a single service, then the aggregator returns that 1 service's metadata. @@ -259,8 +261,8 @@ def test_list_services_simple( assert actual_services == expected_services def test_list_services_merged( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, - service_metadata_wmts_foo, service_metadata_wms_bar + self, flask_app, multi_backend_connection, config, catalog, + backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """Given 2 backends with each 1 service, then the aggregator lists both services.""" @@ -278,8 +280,8 @@ def test_list_services_merged( assert sorted(actual_services) == sorted(expected_services) def test_list_services_merged_multiple( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, - service_metadata_wmts_foo, service_metadata_wms_bar + self, flask_app, multi_backend_connection, config, catalog, + backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """Given multiple services across 2 backends, the aggregator lists all service types from all backends.""" services1 = { @@ -343,8 +345,8 @@ def test_list_services_merged_multiple( assert sorted(actual_services) == sorted(expected_services) def test_service_info_succeeds( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, backend2, requests_mock, - service_metadata_wmts_foo, service_metadata_wms_bar + self, flask_app, multi_backend_connection, config, catalog, + backend1, backend2, requests_mock, service_metadata_wmts_foo, service_metadata_wms_bar ): """When it gets a correct service ID, it returns the expected ServiceMetadata.""" json_wmts_foo = service_metadata_wmts_foo.prepare_for_json() @@ -374,7 +376,7 @@ def test_service_info_succeeds( assert actual_service2 == expected_service2 def test_service_info_wrong_backend_id( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock, + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock, service_metadata_wmts_foo ): """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" @@ -388,7 +390,7 @@ def test_service_info_wrong_backend_id( implementation.service_info(user_id=TEST_USER, service_id="backenddoesnotexist-wtms-foo") def test_service_info_wrong_service_id( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock, + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock, ): """When it gets a non-existent service ID, it raises a ServiceNotFoundException.""" @@ -403,7 +405,7 @@ def test_service_info_wrong_service_id( assert requests_mock.called def test_create_service_succeeds( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock ): """When it gets a correct params for a new service, it successfully creates it.""" @@ -437,7 +439,7 @@ def test_create_service_succeeds( @pytest.mark.parametrize("exception_class", [OpenEoApiError, OpenEoRestError]) def test_create_service_backend_raises_openeoapiexception( - self, flask_app, api100, multi_backend_connection, config, catalog, + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock, exception_class ): """When the backend raises a general exception the aggregator raises an OpenEOApiException.""" @@ -466,7 +468,7 @@ def test_create_service_backend_raises_openeoapiexception( [ProcessGraphMissingException, ProcessGraphInvalidException, ServiceUnsupportedException] ) def test_create_service_backend_reraises( - self, flask_app, api100, multi_backend_connection, config, catalog, + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock, exception_class ): """When the backend raises exception types that indicate client error / bad input data, @@ -495,7 +497,7 @@ def test_create_service_backend_reraises( ) def test_remove_service_succeeds( - self, flask_app, api100, multi_backend_connection, config, catalog, backend1, requests_mock + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock ): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" @@ -510,7 +512,7 @@ def test_remove_service_succeeds( assert mock_delete.called def test_remove_service_but_backend_id_not_found( - self, flask_app, api100, multi_backend_connection, config, catalog, + self, flask_app, multi_backend_connection, config, catalog, ): """When the backend ID/prefix does not exist then the aggregator raises an ServiceNotFoundException.""" @@ -523,7 +525,7 @@ def test_remove_service_but_backend_id_not_found( implementation.remove_service(user_id=TEST_USER, service_id="doesnotexist-wmts-foo") def test_remove_service_but_service_id_not_found( - self, flask_app, api100, multi_backend_connection, config, catalog, + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock ): """When the service ID does not exist for the specified backend then the aggregator raises an ServiceNotFoundException.""" @@ -544,7 +546,7 @@ def test_remove_service_but_service_id_not_found( assert mock_delete1.called def test_remove_service_backend_response_is_an_error_status( - self, flask_app, api100, multi_backend_connection, config, catalog, + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock ): """When the backend response is an HTTP error status then the aggregator raises an OpenEoApiError.""" @@ -562,7 +564,7 @@ def test_remove_service_backend_response_is_an_error_status( assert e.value.http_status_code == 500 def test_update_service_succeeds( - self, flask_app, api100, multi_backend_connection, config, catalog, + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" @@ -586,7 +588,7 @@ def test_update_service_succeeds( assert mock_patch.last_request.json() == expected_process def test_update_service_but_backend_id_does_not_exist( - self, flask_app, api100, multi_backend_connection, config, catalog, + self, flask_app, multi_backend_connection, config, catalog, ): """When the backend ID/prefix does not exist then the aggregator raises an ServiceNotFoundException.""" @@ -599,8 +601,8 @@ def test_update_service_but_backend_id_does_not_exist( implementation.update_service(user_id=TEST_USER, service_id="doesnotexist-wmts-foo", process_graph=process_graph_after) def test_update_service_but_service_id_not_found( - self, flask_app, api100, multi_backend_connection, config, catalog, - backend1, backend2, requests_mock + self, flask_app, multi_backend_connection, config, catalog, + backend1, requests_mock ): """When the service ID does not exist for the specified backend then the aggregator raises an ServiceNotFoundException.""" @@ -620,7 +622,7 @@ def test_update_service_but_service_id_not_found( assert mock_patch1.called def test_update_service_backend_response_is_an_error_status( - self, flask_app, api100, multi_backend_connection, config, catalog, + self, flask_app, multi_backend_connection, config, catalog, backend1, requests_mock ): """When the backend response is an HTTP error status then the aggregator raises an OpenEoApiError.""" diff --git a/tests/test_views.py b/tests/test_views.py index f3c680e1..bf735bf2 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -1408,9 +1408,7 @@ def test_service_types_merging(self, api100, backend1, backend2, requests_mock): expected_service_types.update(service_type_2) assert actual_service_types == expected_service_types - def test_service_info( - self, api100, backend1, requests_mock - ): + def test_service_info(self, api100, backend1, requests_mock): """When it gets a correct service ID, it returns the expected service's metadata as JSON.""" json_wmts_foo = { @@ -1535,9 +1533,7 @@ def test_create_wmts_reports_500_server_error(self, api100, requests_mock, backe resp = api100.post('/services', json=post_data) assert resp.status_code == 500 - def test_remove_service_succeeds( - self, api100, requests_mock, backend1 - ): + def test_remove_service_succeeds(self, api100, requests_mock, backend1): """When remove_service is called with an existing service ID, it removes service and returns HTTP 204.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) @@ -1559,9 +1555,7 @@ def test_remove_service_but_backend_id_not_found(self, api100): assert resp.status_code == 404 - def test_remove_service_but_service_id_not_found( - self, api100, backend1, requests_mock - ): + def test_remove_service_but_service_id_not_found(self, api100, backend1, requests_mock): """When the service ID does not exist then the aggregator responds with HTTP 404, not found.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) @@ -1576,7 +1570,7 @@ def test_remove_service_but_service_id_not_found( assert mock_delete.called def test_remove_service_backend_response_is_an_error_status( - self, api100, requests_mock, backend1, service_metadata_wmts_foo + self, api100, requests_mock, backend1, service_metadata_wmts_foo ): """When the backend response is an error, HTTP 500, then the aggregator also responds with HTTP 500 status.""" @@ -1605,7 +1599,7 @@ def test_remove_service_backend_response_is_an_error_status( assert mock_delete.called def test_update_service_service_succeeds( - self, api100, backend1, requests_mock, service_metadata_wmts_foo + self, api100, backend1, requests_mock, service_metadata_wmts_foo ): """When it receives an existing service ID and a correct payload, it updates the expected service.""" @@ -1627,9 +1621,7 @@ def test_update_service_service_succeeds( assert mock_patch.last_request.json() == json_payload - def test_update_service_but_backend_id_not_found( - self, api100 - ): + def test_update_service_but_backend_id_not_found(self, api100): """When the service ID does not exist because the backend prefix is wrong, then the aggregator responds with HTTP 404, not found.""" api100.set_auth_bearer_token(TEST_USER_BEARER_TOKEN) @@ -1641,7 +1633,7 @@ def test_update_service_but_backend_id_not_found( assert resp.status_code == 404 def test_update_service_service_id_not_found( - self, api100, backend1, requests_mock, service_metadata_wmts_foo + self, api100, backend1, requests_mock, service_metadata_wmts_foo ): """When the service ID does not exist for the specified backend, then the aggregator responds with HTTP 404, not found."""