From 15d448b34e56225c1a3df29332da776d0242469d Mon Sep 17 00:00:00 2001 From: Jarrett Keifer Date: Wed, 11 Dec 2024 15:26:03 -0300 Subject: [PATCH] Split constraints and opportunity properties into two concepts (#113) * split constraints/opportunity properties * add better documentation * add json schema idea to constraints adr --------- Co-authored-by: Phil Varner --- CHANGELOG.md | 29 ++++++ README.md | 15 ++- adrs/README.md | 3 + adrs/constraints.md | 100 ++++++++++++++++++++ src/stapi_fastapi/models/opportunity.py | 1 + src/stapi_fastapi/models/product.py | 18 +++- src/stapi_fastapi/routers/product_router.py | 5 +- bin/server.py => tests/application.py | 24 ++++- tests/conftest.py | 28 +++--- tests/shared.py | 12 --- tests/test_order.py | 5 +- tests/test_product.py | 1 - 12 files changed, 203 insertions(+), 38 deletions(-) create mode 100644 adrs/README.md create mode 100644 adrs/constraints.md rename bin/server.py => tests/application.py (87%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c11c2..4f194d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,35 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [unreleased] + +### Added + +none + +### Changed + +- The concepts of Opportunity search Constraint and Opportunity search result Opportunity Properties are now separate, + recognizing that they have related attributes, but not neither the same attributes or the same values for those attributes. + +### Deprecated + +none + +### Removed + +none + +### Fixed + +none + +### Security + +none + + ## [0.3.0] - 2024-12-6 ### Added diff --git a/README.md b/README.md index d62ab17..e445061 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # STAPI FastAPI - Sensor Tasking API with FastAPI -WARNING: The whole [STAPI spec] is very much work in progress, so things are +WARNING: The whole [STAPI spec] is very much a work in progress, so things are guaranteed to be not correct. ## Usage @@ -8,6 +8,11 @@ guaranteed to be not correct. STAPI FastAPI provides an `fastapi.APIRouter` which must be included in `fastapi.FastAPI` instance. + +## ADRs + +ADRs can be found in in the [adrs](./adrs/README.md) directory. + ## Development It's 2024 and we still need to pick our poison for a 2024 dependency management @@ -36,12 +41,12 @@ command `pytest`. This project cannot be run on its own because it does not have any backend implementations. However, a minimal test implementation is provided in -[`./bin/server.py`](./bin/server.py). It can be run with `uvicorn` as a way to -interact with the API and to view the OpenAPI documentation. Run it like so -from the project root: +[`./tests/application.py`](./tests/application.py). It can be run with +`uvicorn` as a way to interact with the API and to view the OpenAPI +documentation. Run it like so from the project root: ```commandline -uvicorn server:app --app-dir ./bin --reload +uvicorn application:app --app-dir ./tests --reload ``` With the `uvicorn` defaults the app should be accessible at diff --git a/adrs/README.md b/adrs/README.md new file mode 100644 index 0000000..e68819c --- /dev/null +++ b/adrs/README.md @@ -0,0 +1,3 @@ +# ADRs + +- [Constraints and Opportunity Properties](./constraints.md) diff --git a/adrs/constraints.md b/adrs/constraints.md new file mode 100644 index 0000000..af51296 --- /dev/null +++ b/adrs/constraints.md @@ -0,0 +1,100 @@ +# Constraints and Opportunity Properties + +Previously, the Constraints and Opportunity Properties were the same concept/representation. However, these represent distinct but related attributes. Constraints represents the terms that can be used in the filter sent to the Opportunities Search and Order Create endpoints. These are frequently the same or related values that will be part of the STAC Items that are used to fulfill an eventual Order. Opportunity Properties represent the expected range of values that these STAC Items are expected to have. An opportunity is a prediction about the future, and as such, the values for the Opportunity are fuzzy. For example, the sun azimuth angle will (likely) be within a predictable range of values, but the exact value will not be known until after the capture occurs. Therefore, it is necessary to describe the Opportunity in a way that describes these ranges. + +For example, for the concept of "off_nadir": + +The Constraint will be a term "off_nadir" that can be a value 0 to 45. +This is used in a CQL2 filter to the Opportunities Search endpoint to restrict the allowable values from 0 to 15 +The Opportunity that is returned from Search has an Opportunity Property "off_nadir" with a description that the value of this field in the resulting STAC Items will be between 4 and 8, which falls within the filter restriction of 0-15. +An Order is created with the original filter and other fields. +The Order is fulfilled with a STAC Item that has an off_nadir value of 4.8. + +As of Dec 2024, the STAPI spec says only that the Opportunity Properties must have a datetime interval field `datetime` and a `product_id` field. The remainder of the Opportunity description proprietary is up to the provider to define. The example given this this repo for `off_nadir` is of a custom format with a "minimum" and "maximum" field describing the limits. + +## JSON Schema + +Another option would be to use either a full JSON Schema definition for an attribute value in the properties (e.g., `schema`) or individual attribute definitions for the properties values. This option should be investigated further in the future. + +JSON Schema is a well-defined specification language that can support this type of data description. It is already used as the language for OGC API Queryables to define the constraints on various terms that may be used in CQL2 expressions, and likewise within STAPI for the Constraints that are used in Opportunity Search and the Order Parameters that are set on an order. The use of JSON Schema for Constraints (as with Queryables) is not to specify validation for a JSON document, but rather to well-define a set of typed and otherwise-constrained terms. Similarly, JSON Schema would be used for the Opportunity to define the predicted ranges of properties within the Opportunity that is bound to fulfill an Order. + +The geometry is not one of the fields that will be expressed as a schema constraint, since this is part of the Opportunity/Item/Feature top-level. The Opportunity geometry will express both uncertainty about the actual capture area and a “maximum extent” of capture, e.g., a small area within a larger data strip – this is intentionally vague so it can be used to express whatever semantics the provider wants. + +The ranges of predicted Opportunity values can be expressed using JSON in the following way: + +- numeric value - number with const, enum, or minimum/maximum/exclusiveMinimum/exclusiveMaximum +- string value - string with const or enum +- datetime - type string using format date-time. The limitation wit this is that these values are not treated with JSON Schema as temporal, but rather a string pattern. As such, there is no formal way to define a temporal interval that the instance value must be within. Instead, we will repurpose the description field as a datetime interval in the same format as a search datetime field, e.g., 2024-01-01T00:00:00Z/2024-01-07T00:00:00Z. Optionally, the pattern field can be defined if the valid datetime values also match a regular expression, e.g., 2024-01-0[123456]T.*, which while not as useful semantically as the description interval does provide a formal validation of the resulting object, which waving hand might be useful in some way waving hand . + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "schema.json", + "type": "object", + "properties": { + "datetime": { + "title": "Datetime", + "type": "string", + "format": "date-time", + "description": "2024-01-01T00:00:00Z/2024-01-07T00:00:00Z", + "pattern": "2024-01-0[123456]T.*" + }, + "sensor_type": { + "title": "Sensor Type", + "type": "string", + "const": "2" + }, + "craft_id": { + "title": "Spacecraft ID", + "type": "string", + "enum": [ + "7", + "8" + ] + }, + "view:sun_elevation": { + "title": "View:Sun Elevation", + "type": "number", + "minimum": 30.0, + "maximum": 35.0 + }, + "view:azimuth": { + "title": "View:Azimuth", + "type": "number", + "exclusiveMinimum": 104.0, + "exclusiveMaximum": 115.0 + }, + "view:off_nadir": { + "title": "View:Off Nadir", + "type": "number", + "minimum": 0.0, + "maximum": 9.0 + }, + "eo:cloud_cover": { + "title": "Eo:Cloud Cover", + "type": "number", + "minimum": 5.0, + "maximum": 15.0 + } + } +} +``` + +The Item that fulfills and Order placed on this Opportunity might be like: + + +```json +{ + "type": "Feature", + ... + "properties": { + "datetime": "2024-01-01T00:00:00Z", + "sensor_type": "2", + "craft_id": "7", + "view:sun_elevation": 30.0, + "view:azimuth": 105.0, + "view:off_nadir": 9.0, + "eo:cloud_cover": 10.0 + } +} +``` diff --git a/src/stapi_fastapi/models/opportunity.py b/src/stapi_fastapi/models/opportunity.py index 0bb7b13..930cfcf 100644 --- a/src/stapi_fastapi/models/opportunity.py +++ b/src/stapi_fastapi/models/opportunity.py @@ -12,6 +12,7 @@ # Copied and modified from https://github.com/stac-utils/stac-pydantic/blob/main/stac_pydantic/item.py#L11 class OpportunityProperties(BaseModel): datetime: DatetimeInterval + product_id: str model_config = ConfigDict(extra="allow") diff --git a/src/stapi_fastapi/models/product.py b/src/stapi_fastapi/models/product.py index 50f8636..bf5118a 100644 --- a/src/stapi_fastapi/models/product.py +++ b/src/stapi_fastapi/models/product.py @@ -13,6 +13,9 @@ from stapi_fastapi.backends.product_backend import ProductBackend +type Constraints = BaseModel + + class ProviderRole(StrEnum): licensor = "licensor" producer = "producer" @@ -28,7 +31,7 @@ class Provider(BaseModel): # redefining init is a hack to get str type to validate for `url`, # as str is ultimately coerced into an AnyHttpUrl automatically anyway - def __init__(self, url: AnyHttpUrl | str, **kwargs): + def __init__(self, url: AnyHttpUrl | str, **kwargs) -> None: super().__init__(url=url, **kwargs) @@ -44,7 +47,8 @@ class Product(BaseModel): links: list[Link] = Field(default_factory=list) # we don't want to include these in the model fields - _constraints: type[OpportunityProperties] + _constraints: type[Constraints] + _opportunity_properties: type[OpportunityProperties] _order_parameters: type[OrderParameters] _backend: ProductBackend @@ -52,13 +56,15 @@ def __init__( self, *args, backend: ProductBackend, - constraints: type[OpportunityProperties], + constraints: type[Constraints], + opportunity_properties: type[OpportunityProperties], order_parameters: type[OrderParameters], **kwargs, ) -> None: super().__init__(*args, **kwargs) self._backend = backend self._constraints = constraints + self._opportunity_properties = opportunity_properties self._order_parameters = order_parameters @property @@ -66,9 +72,13 @@ def backend(self: Self) -> ProductBackend: return self._backend @property - def constraints(self: Self) -> type[OpportunityProperties]: + def constraints(self: Self) -> type[Constraints]: return self._constraints + @property + def opportunity_properties(self: Self) -> type[OpportunityProperties]: + return self._opportunity_properties + @property def order_parameters(self: Self) -> type[OrderParameters]: return self._order_parameters diff --git a/src/stapi_fastapi/routers/product_router.py b/src/stapi_fastapi/routers/product_router.py index d17b91d..c628986 100644 --- a/src/stapi_fastapi/routers/product_router.py +++ b/src/stapi_fastapi/routers/product_router.py @@ -51,7 +51,10 @@ def __init__( methods=["POST"], response_class=GeoJSONResponse, # unknown why mypy can't see the constraints property on Product, ignoring - response_model=OpportunityCollection[Geometry, self.product.constraints], # type: ignore + response_model=OpportunityCollection[ + Geometry, + self.product.opportunity_properties, # type: ignore + ], summary="Search Opportunities for the product", tags=["Products"], ) diff --git a/bin/server.py b/tests/application.py similarity index 87% rename from bin/server.py rename to tests/application.py index e379657..9d7c2ea 100644 --- a/bin/server.py +++ b/tests/application.py @@ -1,7 +1,9 @@ from datetime import datetime, timezone +from typing import Literal, Self from uuid import uuid4 from fastapi import FastAPI, Request +from pydantic import BaseModel, Field, model_validator from returns.maybe import Maybe from returns.result import Failure, ResultE, Success @@ -108,10 +110,27 @@ async def create_order( return Failure(e) -class MyOpportunityProperties(OpportunityProperties): +class MyProductConstraints(BaseModel): off_nadir: int +class OffNadirRange(BaseModel): + minimum: int = Field(ge=0, le=45) + maximum: int = Field(ge=0, le=45) + + @model_validator(mode="after") + def validate_range(self) -> Self: + if self.minimum > self.maximum: + raise ValueError("range minimum cannot be greater than maximum") + return self + + +class MyOpportunityProperties(OpportunityProperties): + off_nadir: OffNadirRange + vehicle_id: list[Literal[1, 2, 5, 7, 8]] + platform: Literal["platform_id"] + + class MyOrderParameters(OrderParameters): s3_path: str | None = None @@ -135,7 +154,8 @@ class MyOrderParameters(OrderParameters): keywords=["test", "satellite"], providers=[provider], links=[], - constraints=MyOpportunityProperties, + constraints=MyProductConstraints, + opportunity_properties=MyOpportunityProperties, order_parameters=MyOrderParameters, backend=product_backend, ) diff --git a/tests/conftest.py b/tests/conftest.py index 9781fc3..56a00ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,19 +14,21 @@ Opportunity, ) from stapi_fastapi.models.product import ( - OrderParameters, Product, Provider, ProviderRole, ) from stapi_fastapi.routers.root_router import RootRouter -from .backends import MockOrderDB, MockProductBackend, MockRootBackend -from .shared import SpotlightOpportunityProperties, SpotlightOrderParameters, find_link - - -class TestSpotlightOrderParameters(OrderParameters): - s3_path: str | None = None +from .application import ( + MockOrderDB, + MockProductBackend, + MockRootBackend, + MyOpportunityProperties, + MyOrderParameters, + MyProductConstraints, +) +from .shared import find_link @pytest.fixture(scope="session") @@ -62,8 +64,9 @@ def mock_product_test_spotlight( keywords=["test", "satellite"], providers=[mock_provider], links=[], - constraints=SpotlightOpportunityProperties, - order_parameters=SpotlightOrderParameters, + constraints=MyProductConstraints, + opportunity_properties=MyOpportunityProperties, + order_parameters=MyOrderParameters, backend=product_backend, ) @@ -140,10 +143,13 @@ def mock_test_spotlight_opportunities() -> list[Opportunity]: type="Point", coordinates=Position2D(longitude=0.0, latitude=0.0), ), - properties=SpotlightOpportunityProperties( + properties=MyOpportunityProperties( product_id="xyz123", datetime=(start, end), - off_nadir=20, + off_nadir={"minimum": 20, "maximum": 22}, + vehicle_id=[1], + platform="platform_id", + other_thing="abcd1234", ), ), ] diff --git a/tests/shared.py b/tests/shared.py index 3260feb..4a2aa32 100644 --- a/tests/shared.py +++ b/tests/shared.py @@ -1,17 +1,5 @@ from typing import Any -from stapi_fastapi.models.opportunity import OpportunityProperties -from stapi_fastapi.models.order import OrderParameters - - -class SpotlightOpportunityProperties(OpportunityProperties): - off_nadir: int - - -class SpotlightOrderParameters(OrderParameters): - s3_path: str | None = None - - type link_dict = dict[str, Any] diff --git a/tests/test_order.py b/tests/test_order.py index 2e14903..41edd17 100644 --- a/tests/test_order.py +++ b/tests/test_order.py @@ -9,8 +9,9 @@ from stapi_fastapi.models.order import OrderRequest +from .application import MyOrderParameters from .backends import MockProductBackend -from .shared import SpotlightOrderParameters, find_link +from .shared import find_link NOW = datetime.now(UTC) START = NOW @@ -29,7 +30,7 @@ def create_order_allowed_payloads() -> list[OrderRequest]: datetime.fromisoformat("2024-11-15T18:55:33Z"), ), filter=None, - order_parameters=SpotlightOrderParameters(s3_path="s3://my-bucket"), + order_parameters=MyOrderParameters(s3_path="s3://my-bucket"), ), ] diff --git a/tests/test_product.py b/tests/test_product.py index 71c5783..ac8e0cf 100644 --- a/tests/test_product.py +++ b/tests/test_product.py @@ -48,7 +48,6 @@ def test_product_constraints_response( json_schema = res.json() assert "properties" in json_schema - assert "datetime" in json_schema["properties"] assert "off_nadir" in json_schema["properties"]