From 8f1ab3b37aebbd69c7d4f70d12a5952ec826fb62 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 5 Mar 2025 16:17:32 +0100 Subject: [PATCH] Issue #22 add online/offline status to federation overview --- CHANGELOG.md | 2 +- src/openeo_aggregator/backend.py | 10 ++------ src/openeo_aggregator/config/config.py | 1 + src/openeo_aggregator/connection.py | 34 ++++++++++++++++++-------- tests/conftest.py | 13 ++++++++-- tests/test_connection.py | 29 +++++++++++++++++----- tests/test_views.py | 2 ++ 7 files changed, 64 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 279e55e..6ca13b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The format is roughly based on [Keep a Changelog](https://keepachangelog.com/en/ ## unreleased - Bump minimum required Python version to 3.11 ([#127](https://github.com/Open-EO/openeo-aggregator/issues/127), [#174](https://github.com/Open-EO/openeo-aggregator/issues/174)) -- Add title and description to federation listing in capabilities document ([#22](https://github.com/Open-EO/openeo-aggregator/issues/22)) +- Add title, description and online/offline status to federation listing in capabilities document ([#22](https://github.com/Open-EO/openeo-aggregator/issues/22)) ## 0.43.0 diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 15faaf5..34a4ece 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -1596,14 +1596,8 @@ def capabilities_billing(self) -> dict: def postprocess_capabilities(self, capabilities: dict) -> dict: # TODO: which url to use? unversioned or versioned? see https://github.com/Open-EO/openeo-api/pull/419 - capabilities["federation"] = { - bid: { - "url": status["root_url"], - "title": status.get("title") or f"Backend {bid!r}", - "description": status.get("description") or f"OpenEO backend {bid!r}", - } - for bid, status in self._backends.get_status().items() - } + capabilities["federation"] = self._backends.get_federation_overview() + # TODO: standardize this field? capabilities["_partitioned_job_tracking"] = bool(self.batch_jobs.partitioned_job_tracker) return capabilities diff --git a/src/openeo_aggregator/config/config.py b/src/openeo_aggregator/config/config.py index c47012f..3da4700 100644 --- a/src/openeo_aggregator/config/config.py +++ b/src/openeo_aggregator/config/config.py @@ -54,6 +54,7 @@ class AggregatorBackendConfig(OpenEoBackendConfig): packages=["openeo", "openeo_driver", "openeo_aggregator"], ) + # TODO: allow to specify more info per backend in addtion to just URL: title, description, experimental flag, ... aggregator_backends: Dict[str, str] = attrs.field(validator=attrs.validators.min_len(1)) # See `ZooKeeperPartitionedJobDB.from_config` for supported fields. diff --git a/src/openeo_aggregator/connection.py b/src/openeo_aggregator/connection.py index 8d29cdf..e847491 100644 --- a/src/openeo_aggregator/connection.py +++ b/src/openeo_aggregator/connection.py @@ -234,6 +234,7 @@ def __init__( ) # TODO: backend_urls as dict does not have explicit order, while this is important. _log.info(f"Creating MultiBackendConnection with {backends=!r}") + # TODO: support more backend info than just URL (title, description, experimental flag, ...) self._backend_urls = backends self._configured_oidc_providers = configured_oidc_providers @@ -258,7 +259,7 @@ def from_config() -> "MultiBackendConnection": ) def _get_connections(self, skip_failures=False) -> Iterator[BackendConnection]: - """Create new backend connections.""" + """Create new backend connections, possibly skipping connection failures.""" for bid, url in self._backend_urls.items(): try: _log.info(f"Create backend {bid!r} connection to {url!r}") @@ -313,20 +314,33 @@ def get_connection(self, backend_id: str) -> BackendConnection: return con raise OpenEOApiException(f"No backend with id {backend_id!r}") - def get_status(self) -> dict: - # TODO: reconsider method name (currently it has little to do with status) - return { + def get_federation_overview(self) -> dict: + """ + Federation overview, to be used in the capabilities document (`GET /`), + under the "federation" field, per federation extension + (conformance class https://api.openeo.org/extensions/federation/0.1.0) + """ + federation = { c.id: { - # TODO: avoid private attributes? - # TODO: add real backend status? (cached?) - "root_url": c._root_url, - "orig_url": c._orig_url, - "title": c.capabilities().get("title"), - "description": c.capabilities().get("description"), + "url": c.root_url, + "title": c.capabilities().get("title", f"Backend {c.id!r}"), + "description": c.capabilities().get("description", f"Federated openEO backend {c.id!r}"), + "status": "online", } for c in self.get_connections() } + for bid, url in self._backend_urls.items(): + if bid not in federation: + federation[bid] = { + "url": url, + "status": "offline", + "title": f"Backend {bid!r}", + "description": f"Federated openEO backend {bid!r}", + } + + return federation + def _get_api_versions(self) -> List[str]: return list(set(c.capabilities().api_version() for c in self.get_connections())) diff --git a/tests/conftest.py b/tests/conftest.py index 0114a11..c65a9d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,7 +44,11 @@ def backend1(requests_mock, mbldr) -> str: domain = "https://b1.test/v1" # TODO: how to work with different API versions? requests_mock.get( - domain + "/", json=mbldr.capabilities(title="Dummy Federation One", description="Welcome to Federation One.") + domain + "/", + json=mbldr.capabilities( + title="Dummy Federation One", + description="Welcome to Federation One.", + ), ) requests_mock.get(domain + "/credentials/oidc", json=mbldr.credentials_oidc()) requests_mock.get(domain + "/processes", json=mbldr.processes(*_DEFAULT_PROCESSES)) @@ -54,7 +58,12 @@ def backend1(requests_mock, mbldr) -> str: @pytest.fixture def backend2(requests_mock, mbldr) -> str: domain = "https://b2.test/v1" - requests_mock.get(domain + "/", json=mbldr.capabilities(title="Dummy The Second")) + requests_mock.get( + domain + "/", + json=mbldr.capabilities( + title="Dummy The Second", + ), + ) requests_mock.get(domain + "/credentials/oidc", json=mbldr.credentials_oidc()) requests_mock.get(domain + "/processes", json=mbldr.processes(*_DEFAULT_PROCESSES)) return domain diff --git a/tests/test_connection.py b/tests/test_connection.py index 247b063..c207658 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -267,19 +267,36 @@ def test_map(self, multi_backend_connection, backend1, backend2, requests_mock): assert isinstance(res, types.GeneratorType) assert list(res) == [("b1", {"bar": 1}), ("b2", {"meh": 2})] - def test_status(self, multi_backend_connection): - assert multi_backend_connection.get_status() == { + def test_get_federation_overview_basic(self, multi_backend_connection): + assert multi_backend_connection.get_federation_overview() == { "b1": { - "orig_url": "https://b1.test/v1", - "root_url": "https://b1.test/v1", + "url": "https://b1.test/v1", "title": "Dummy Federation One", "description": "Welcome to Federation One.", + "status": "online", }, "b2": { - "orig_url": "https://b2.test/v1", - "root_url": "https://b2.test/v1", + "url": "https://b2.test/v1", "title": "Dummy The Second", "description": "Test instance of openEO Aggregator", + "status": "online", + }, + } + + def test_get_federation_overview_offline(self, multi_backend_connection, backend1, backend2, requests_mock): + requests_mock.get(f"{backend2}/", status_code=500, json={"error": "nope"}) + assert multi_backend_connection.get_federation_overview() == { + "b1": { + "url": "https://b1.test/v1", + "title": "Dummy Federation One", + "description": "Welcome to Federation One.", + "status": "online", + }, + "b2": { + "url": "https://b2.test/v1", + "description": "Federated openEO backend 'b2'", + "title": "Backend 'b2'", + "status": "offline", }, } diff --git a/tests/test_views.py b/tests/test_views.py index 1e6972f..51f516c 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -52,11 +52,13 @@ def test_capabilities(self, api100): "url": "https://b1.test/v1", "title": "Dummy Federation One", "description": "Welcome to Federation One.", + "status": "online", }, "b2": { "url": "https://b2.test/v1", "title": "Dummy The Second", "description": "Test instance of openEO Aggregator", + "status": "online", }, }