Merge pull request #11 from AustralianSynchrotron/SCOMPMX-627-deploy-…

…core-mx3-services-to-kubernetes

Scompmx 627 deploy core mx3 services to kubernetes

.gitignore

@@ -1,3 +1,136 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Deployment
+/deployment/*secret*
+
+# IDE
+/.venv/
+/.poetry_env/
 .vscode/
+.idea/
+
+
 *.h5
-__pycache__/

.pre-commit-config.yaml

@@ -48,4 +48,4 @@ repos:
     hooks:
     - id: flake8
       name: Flake8
-      additional_dependencies: ["flake8-bugbear==20.1.4"]
+      additional_dependencies: ["flake8-bugbear==24.8.19"]

Dockerfile

@@ -1,6 +1,6 @@
-FROM python:3.11
+FROM python:3.12
 
-ENV HDF5_MASTER_FILE=/share_data/datasets_full/minimalInsulinMX1/030/testcrystal_0014_master.h5
+ENV AS_HDF5_MASTER_FILE=/share_data/datasets_full/minimalInsulinMX1/030/testcrystal_0014_master.h5
 ENV POETRY_VERSION=1.8.3
 
 # Install OS packages

README.md

@@ -1,17 +1,17 @@
 # ANSTO Sim-Plon API
 A simulated Dectris Simplon API. Aims to have the same RESTlike interface and produce a ZMQ stream of data from an input Hdf5 file.
-The input HDF5 file is defined by the environment variable `HDF5_MASTER_FILE`, e.g. `HDF5_MASTER_FILE=/path/to/HDF5_masterfile.h5`
+The input HDF5 file is defined by the environment variable `AS_HDF5_MASTER_FILE`, e.g. `AS_HDF5_MASTER_FILE=/path/to/HDF5_masterfile.h5`
 
 Currently generates a [Stream V2] release compatible ZMQ stream.
 
 ## Setup
 
 **Simulated Simplon API Configuration**
 
-   To run the simulated Simplon API, you need to specify the path of an HDF5 master file using the `HDF5_MASTER_FILE` environment variable. You can also configure other parameters using the following environment variables:
+   To run the simulated Simplon API, you need to specify the path of an HDF5 master file using the `AS_HDF5_MASTER_FILE` environment variable. You can also configure other parameters using the following environment variables:
 
-   - `DELAY_BETWEEN_FRAMES`: Specifies the delay between frames in seconds (default: 0.01 s). This number can be modified via the `/ansto_endpoints/delay_between_frames` endpoint.
-   - `NUMBER_OF_DATA_FILES`: Sets the number of data files from the master file loaded into memory (default: 1). The number of datafiles can be additionally modified when loading a new master file using the
+   - `AS_DELAY_BETWEEN_FRAMES`: Specifies the delay between frames in seconds (default: 0.01 s). This number can be modified via the `/ansto_endpoints/delay_between_frames` endpoint.
+   - `AS_NUMBER_OF_DATA_FILES`: Sets the number of data files from the master file loaded into memory (default: 1). The number of datafiles can be additionally modified when loading a new master file using the
    `/ansto_endpoints/hdf5_master_file` endpoint.
    - The number of frames per trigger is set automatically to the number of frames in the master file. This can be modified by using the `/detector/api/1.8.0/config/nimages` endpoint.
 
@@ -34,7 +34,7 @@ Follow these steps to run the simulated SIMPLON API and ensure its proper functi
 
 2. **Set the HDF5 File Path**
 
-   Before running the API, set the `HDF5_MASTER_FILE` environment variable using the following command:
+   Before running the API, set the `AS_HDF5_MASTER_FILE` environment variable using the following command:
    ```bash
    export HDF5_MASTER_FILE=/path/to/HDF5_master_file
    ```
@@ -43,6 +43,12 @@ Follow these steps to run the simulated SIMPLON API and ensure its proper functi
       ```bash
    uvicorn ansto_simplon_api.main:app
    ```
+   > **Note:** The FastAPI Swagger / ReDoc endpoint documentation is disabled by default, to enable the documentation, please set the following environment variables.
+   > ```bash
+   > export AS_API_DOCS_URL=/swagger
+   > export AS_API_REDOC_URL=/docs
+   > export AS_API_OPENAPI_URL=/openapi.json
+   >```
 
 4. **Start the ZMQ Consumer**
 

ansto_simplon_api/__init__.py

@@ -0,0 +1,3 @@
+from ._version import __version__
+
+__all__ = ("__version__",)

ansto_simplon_api/_version.py

@@ -0,0 +1,19 @@
+from importlib.metadata import PackageNotFoundError, version
+
+__version__: str
+try:
+    __version__ = version(__package__ or __name__)
+except PackageNotFoundError:
+    # Could fail to find package when developing locally.
+    # Try reading from pyproject.toml as a backup.
+
+    from pathlib import Path
+    from tomllib import load as load_toml
+
+    try:
+        with open(Path(__file__).resolve().parents[1] / "pyproject.toml", "rb") as f:
+            pyproject = load_toml(f)
+            __version__ = pyproject["tool"]["poetry"]["version"]
+    except OSError:
+        # File IO error
+        __version__ = "local"

ansto_simplon_api/config.py

@@ -0,0 +1,112 @@
+from functools import lru_cache
+from os.path import dirname as os_dirname, join as os_joinpath, realpath as os_realpath
+from pathlib import Path
+from typing import Annotated, Self
+
+from pydantic import Field, FilePath, GetPydanticSchema, SecretStr
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+)
+
+__all__ = ("Settings", "get_settings")
+
+CUR_DIR = os_dirname(os_realpath(__file__))
+
+
+class APISettings(BaseSettings):
+    """API Settings"""
+
+    API_APP_NAME: str = Field(
+        title="App Name",
+        default="Simulated Simplon API",
+    )
+    API_DOCS_URL: str | None = Field(
+        title="OpenAPI Docs Path",
+        default=None,
+    )
+    API_REDOC_URL: str | None = Field(
+        title="ReDocs Path",
+        default=None,
+    )
+    API_OPENAPI_URL: str | None = Field(
+        title="OpenAPI Path",
+        default=None,
+    )
+    API_FAVICON: FilePath = Field(
+        title="Favicon Image",
+        default=os_realpath(os_joinpath(CUR_DIR, "./favicon.ico")),
+    )
+    API_KEY: SecretStr | None = Field(
+        title="Service API-Key",
+        default=None,
+    )
+
+
+class ZMQStreamSettings(BaseSettings):
+    """ZMQ Stream Settings"""
+
+    ZMQ_ADDRESS: str = Field(
+        title="ZMQ Address",
+        default="tcp://*:5555",
+    )
+    HDF5_MASTER_FILE: Annotated[
+        str,
+        GetPydanticSchema(lambda _, _h: _h.generate_schema(FilePath)),
+    ] = Field(
+        title="HDF5 Master File",
+    )
+    DELAY_BETWEEN_FRAMES: float = Field(
+        title="Delay Between Frames",
+        default=0.01,
+    )
+    NUMBER_OF_DATA_FILES: int = Field(
+        title="Number of Data Files",
+        default=1,
+    )
+
+
+class Settings(APISettings, ZMQStreamSettings):
+    README: FilePath = Field(
+        title="ReadMe",
+        default=os_realpath(os_joinpath(CUR_DIR, "../README.md")),
+        validate_default=True,
+    )
+    PYPROJECT: FilePath = Field(
+        title="PyProject",
+        default=os_realpath(os_joinpath(CUR_DIR, "../pyproject.toml")),
+        validate_default=True,
+    )
+    BUILD_INFO: Path = Field(
+        title="Build Info",
+        default=os_realpath(os_joinpath(CUR_DIR, "../.build_info.json")),
+        validate_default=True,
+    )
+
+    model_config = SettingsConfigDict(
+        str_strip_whitespace=True,
+        env_prefix="AS_",
+        env_file=".env",
+        validate_assignment=True,
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls: type[Self],
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        # Restrict settings sources to environment variables only
+        return (env_settings, dotenv_settings)
+
+
+@lru_cache()
+def get_settings() -> Settings:
+    """Cache the settings, this allows the settings to be used in dependencies and
+    for overwriting in tests
+    """
+    return Settings()  # pyright: ignore[reportCallIssue]

ansto_simplon_api/main.py

@@ -1,11 +1,46 @@
 from fastapi import FastAPI
+from fastapi.exception_handlers import http_exception_handler
+from fastapi.logger import logger
+from fastapi.responses import FileResponse
+from starlette.exceptions import HTTPException as StarletteHTTPException
 
+from . import __version__
+from .config import get_settings
 from .routes.ansto_endpoints.load_hdf5_files import router as ansto_endpoints
 from .routes.detector.command import router as command
 from .routes.detector.config import router as detector_config
 from .routes.stream.config import router as stream_config
 
-app = FastAPI()
+config = get_settings()
+
+
+_description: str
+with open(config.README, "r") as _file:
+    _description = _file.read()
+
+
+app = FastAPI(
+    title=config.API_APP_NAME,
+    description=_description,
+    version=__version__,
+    docs_url=config.API_DOCS_URL,
+    redoc_url=config.API_REDOC_URL,
+    openapi_url=config.API_OPENAPI_URL,
+)
+
+
+@app.exception_handler(StarletteHTTPException)
+async def custom_http_exception_handler(request, exc):
+    """Catch any HTTPException and log the error"""
+    logger.error(f"{str(exc)}\n{exc.detail}", exc_info=exc)
+    return await http_exception_handler(request, exc)
+
+
+@app.get("/favicon.ico", include_in_schema=False)
+async def favicon():
+    return FileResponse(config.API_FAVICON)
+
+
 app.include_router(command)
 app.include_router(stream_config)
 app.include_router(detector_config)