Issue #112 Add AggregatorBackendConfig.memoizer

and deprecate `AggregatorConfig.memoizer`

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 The format is roughly based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.25.0]
+
+- Add `AggregatorBackendConfig.memoizer` and deprecate `AggregatorConfig.memoizer` ([#112](https://github.com/Open-EO/openeo-aggregator/issues/112))
+
 ## [0.24.0]
 
 - Add `AggregatorBackendConfig.partitioned_job_tracking` and deprecate `AggregatorConfig.partitioned_job_tracking` ([#112](https://github.com/Open-EO/openeo-aggregator/issues/112))

conf/aggregator.dev.py

@@ -45,6 +45,24 @@
 aggregator_config = AggregatorConfig(
     config_source=__file__,
     zookeeper_prefix="/openeo/aggregator-dev/",
+)
+
+
+config = AggregatorBackendConfig(
+    id="openeo-platform-aggregator-dev",
+    capabilities_title="openEO Platform (dev)",
+    capabilities_description="openEO Platform, provided through openEO Aggregator Driver (development instance).",
+    oidc_providers=oidc_providers,
+    aggregator_backends={
+        "vito": "https://openeo-dev.vito.be/openeo/1.1/",
+        "eodc": "https://openeo-dev.eodc.eu/openeo/1.1.0/",
+        "creo": "https://openeo-staging.creo.vito.be/openeo/1.1",
+        # Sentinel Hub OpenEO by Sinergise
+        "sentinelhub": "https://openeo.sentinel-hub.com/production/",
+    },
+    partitioned_job_tracking={
+        "zk_hosts": ZK_HOSTS,
+    },
     memoizer={
         # See `memoizer_from_config` for more details
         "type": "chained",
@@ -65,21 +83,3 @@
         }
     },
 )
-
-
-config = AggregatorBackendConfig(
-    id="openeo-platform-aggregator-dev",
-    capabilities_title="openEO Platform (dev)",
-    capabilities_description="openEO Platform, provided through openEO Aggregator Driver (development instance).",
-    oidc_providers=oidc_providers,
-    aggregator_backends={
-        "vito": "https://openeo-dev.vito.be/openeo/1.1/",
-        "eodc": "https://openeo-dev.eodc.eu/openeo/1.1.0/",
-        "creo": "https://openeo-staging.creo.vito.be/openeo/1.1",
-        # Sentinel Hub OpenEO by Sinergise
-        "sentinelhub": "https://openeo.sentinel-hub.com/production/",
-    },
-    partitioned_job_tracking={
-        "zk_hosts": ZK_HOSTS,
-    },
-)

conf/aggregator.prod.py

@@ -38,6 +38,23 @@
 aggregator_config = AggregatorConfig(
     config_source=__file__,
     zookeeper_prefix="/openeo/aggregator/",
+)
+
+
+config = AggregatorBackendConfig(
+    id="openeo-platform-aggregator-prod",
+    capabilities_title="openEO Platform",
+    capabilities_description="openEO Platform, provided through openEO Aggregator Driver.",
+    oidc_providers=oidc_providers,
+    aggregator_backends={
+        "vito": "https://openeo.vito.be/openeo/1.1/",
+        "eodc": "https://openeo.eodc.eu/openeo/1.1.0/",
+        # Sentinel Hub OpenEO by Sinergise
+        "sentinelhub": "https://openeo.sentinel-hub.com/production/",
+    },
+    partitioned_job_tracking={
+        "zk_hosts": ZK_HOSTS,
+    },
     memoizer={
         # See `memoizer_from_config` for more details
         "type": "chained",
@@ -57,22 +74,4 @@
             ]
         }
     },
-
-)
-
-
-config = AggregatorBackendConfig(
-    id="openeo-platform-aggregator-prod",
-    capabilities_title="openEO Platform",
-    capabilities_description="openEO Platform, provided through openEO Aggregator Driver.",
-    oidc_providers=oidc_providers,
-    aggregator_backends={
-        "vito": "https://openeo.vito.be/openeo/1.1/",
-        "eodc": "https://openeo.eodc.eu/openeo/1.1.0/",
-        # Sentinel Hub OpenEO by Sinergise
-        "sentinelhub": "https://openeo.sentinel-hub.com/production/",
-    },
-    partitioned_job_tracking={
-        "zk_hosts": ZK_HOSTS,
-    },
 )

src/openeo_aggregator/about.py

@@ -2,7 +2,7 @@
 import sys
 from typing import Optional
 
-__version__ = "0.24.0a1"
+__version__ = "0.25.0a1"
 
 
 def log_version_info(logger: Optional[logging.Logger] = None):

src/openeo_aggregator/caching.py

@@ -530,7 +530,8 @@ def get_memoizer(memoizer_type: str, memoizer_conf: dict) -> Memoizer:
         else:
             raise ValueError(memoizer_type)
 
+    memoizer_config = get_backend_config().memoizer or config.memoizer
     return get_memoizer(
-        memoizer_type=config.memoizer.get("type", "null"),
-        memoizer_conf=config.memoizer.get("config", {}),
+        memoizer_type=memoizer_config.get("type", "null"),
+        memoizer_conf=memoizer_config.get("config", {}),
     )

src/openeo_aggregator/config.py

@@ -43,14 +43,16 @@ class AggregatorConfig(dict):
     config_source = dict_item()
 
     # Dictionary mapping backend id to backend url
-    aggregator_backends = dict_item()  # TODO #112 deprecated
+    # TODO #112 deprecated, instead use by AggregatorBackendConfig.aggregator_backends
+    aggregator_backends = dict_item()
 
-    partitioned_job_tracking = dict_item(default=None)  # TODO #112 deprecated
+    # TODO #112 deprecated, instead use AggregatorBackendConfig.partitioned_job_tracking
+    partitioned_job_tracking = dict_item(default=None)
 
     # TODO #112 Deprecated, use AggregatorBackendConfig.zookeeper_prefix instead
     zookeeper_prefix = dict_item(default="/openeo-aggregator/")
 
-    # See `memoizer_from_config` for details.
+    # TODO #112 Deprecated, use AggregatorBackendConfig.memoizer instead
     memoizer = dict_item(default={"type": "dict"})
 
     # Just a config field for test purposes (while were stripping down this config class)
@@ -147,6 +149,10 @@ class AggregatorBackendConfig(OpenEoBackendConfig):
     #       To be replaced eventually with  "/openeo-aggregator/"
     zookeeper_prefix: str = ""
 
+    # See `memoizer_from_config` for details.
+    # TODO #112: empty default is to allow migration. Te be replaced with `attrs.Factory(lambda: {"type": "dict"})`
+    memoizer: Dict = attrs.Factory(dict)
+
     zk_memoizer_tracking: bool = smart_bool(os.environ.get("OPENEO_AGGREGATOR_ZK_MEMOIZER_TRACKING"))
 
 

tests/backend_config.py

@@ -19,4 +19,8 @@
     },
     connections_cache_ttl=1.0,
     zookeeper_prefix="/o-a/",
+    memoizer={
+        "type": "dict",
+        "config": {"default_ttl": 66},
+    },
 )

tests/background/test_prime_caches.py

@@ -20,17 +20,18 @@
 }
 
 
-@pytest.fixture
-def config(backend1, backend2, backend1_id, backend2_id, zk_client) -> AggregatorConfig:
-    conf = AggregatorConfig()
-    conf.memoizer = {
-        "type": "zookeeper",
-        "config": {
-            "zk_hosts": "zk.test:2181",
-            "default_ttl": 24 * 60 * 60,
-        },
-    }
-    return conf
+@pytest.fixture(autouse=True)
+def _use_zookeeper_memoizer():
+    with config_overrides(
+        memoizer={
+            "type": "zookeeper",
+            "config": {
+                "zk_hosts": "zk.test:2181",
+                "default_ttl": 24 * 60 * 60,
+            },
+        }
+    ):
+        yield
 
 
 @pytest.fixture(autouse=True)

tests/conftest.py

@@ -68,29 +68,15 @@ def zk_client() -> DummyKazooClient:
     return DummyKazooClient()
 
 
-DEFAULT_MEMOIZER_CONFIG = {
-    "type": "dict",
-    "config": {"default_ttl": 66},
-}
 
 
 @pytest.fixture
-def memoizer_config() -> dict:
-    """
-    Fixture for global memoizer config, to allow overriding/parameterizing it for certain tests.
-    Also see https://docs.pytest.org/en/7.1.x/how-to/fixtures.html#override-a-fixture-with-direct-test-parametrization
-    """
-    return DEFAULT_MEMOIZER_CONFIG
-
-
-@pytest.fixture
-def base_config(zk_client, memoizer_config) -> AggregatorConfig:
+def base_config(zk_client) -> AggregatorConfig:
     """Base config for tests (without any configured backends)."""
     conf = AggregatorConfig()
     conf.config_source = "test fixture base_config"
     # conf.flask_error_handling = False  # Temporary disable flask error handlers to simplify debugging (better stack traces).
 
-    conf.memoizer = memoizer_config
 
     return conf