release: v1.4.0 (#61)

* fix: correcting unittest import statement in tests * fix: test organization * fix: optimizing container build for smaller product * fix: updating deps * fix: updating extents calculation * feat: output structured logs with job context * feat: new standardized metrics with dimensions * fix: bumping up version --------- Co-authored-by: drduhe <drduhe@amazon.com> Co-authored-by: jtblack <jtblack@amazon.com> Co-authored-by: Jonathan Blackstock <137826526+jtblack-aws@users.noreply.github.com> Co-authored-by: edparris <edparris@gmail.com>
aws-solutions-library-samples · Feb 16, 2024 · 014eac6 · 014eac6
1 parent f14f1e6
commit 014eac6
Show file tree

Hide file tree

Showing 42 changed files with 692 additions and 454 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,13 +1,12 @@
 *~
 *#
 *.swp
-**/*.iml
+**/*Dockerfile
 **/*.DS_Store
 **/*.tox
 **/*.idea
 **/*venv
 **/__pycache__/
-**/*.py[cod]
 **/*$py.class
 **/*.egg-info/
 **/.coverage
@@ -17,5 +16,4 @@
 **/.mypy_cache
 **/doc/_apidoc/
 **/build
-**/dist
 **/htmlcov
diff --git a/.github/workflows/python-tox.yml b/.github/workflows/python-tox.yml
@@ -13,10 +13,10 @@ jobs:
     - uses: actions/checkout@v4
       with:
         lfs: 'true'
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v3
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.11"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip tox

diff --git a/Dockerfile b/Dockerfile
@@ -1,48 +1,64 @@
-FROM public.ecr.aws/amazonlinux/amazonlinux:2023 as model_runner
+# Stage 1: Build environment
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023-minimal as build
 
-# only override if you're using a mirror with a cert pulled in using cert-base as a build parameter
+# Set up build arguments and environment variables
 ARG BUILD_CERT=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
 ARG PIP_INSTALL_LOCATION=https://pypi.org/simple/
+ARG MINICONDA_VERSION=Miniconda3-latest-Linux-x86_64
+ARG MINICONDA_URL=https://repo.anaconda.com/miniconda/${MINICONDA_VERSION}.sh
+ENV PATH=/opt/conda/bin:$PATH
+ENV CONDA_TARGET_ENV=osml_model_runner
 
-# give sudo permissions
-USER root
-
-# set working directory to home
+# Set working directory
 WORKDIR /home
 
-# configure, update, and refresh yum enviornment
-RUN yum update -y && yum clean all && yum makecache && yum install -y wget shadow-utils
+# Install necessary packages
+USER root
+RUN dnf update -y && dnf install -y wget shadow-utils gcc && dnf clean all
+
+# Install Miniconda
+RUN wget -c ${MINICONDA_URL} && \
+    chmod +x ${MINICONDA_VERSION}.sh && \
+    ./${MINICONDA_VERSION}.sh -b -f -p /opt/conda && \
+    rm ${MINICONDA_VERSION}.sh && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh
+
+# Copy the conda environment file
+COPY environment-py311.yml environment.yml
+
+# Create the conda environment and remove additional unnecessary files
+RUN conda env create -f environment.yml \
+     && conda clean -afy \
+     && find /opt/conda/ -follow -type f -name '*.a' -delete \
+     && find /opt/conda/ -follow -type f -name '*.pyc' -delete \
+     && find /opt/conda/ -follow -type f -name '*.js.map' -delete
+
+# Copy the application source code
+COPY . osml-model-runner
 
-# install miniconda
-ARG MINICONDA_VERSION=Miniconda3-latest-Linux-x86_64
-ARG MINICONDA_URL=https://repo.anaconda.com/miniconda/${MINICONDA_VERSION}.sh
-RUN wget -c ${MINICONDA_URL} \
-    && chmod +x ${MINICONDA_VERSION}.sh \
-    && ./${MINICONDA_VERSION}.sh -b -f -p /opt/conda \
-    && rm ${MINICONDA_VERSION}.sh \
-    && ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh
+# Install the model runner application
+RUN . /opt/conda/etc/profile.d/conda.sh && \
+    conda activate ${CONDA_TARGET_ENV} && \
+    python3 -m pip install osml-model-runner/.
 
-# add conda to the path so we can execute it by name
-ENV PATH=/opt/conda/bin:$PATH
+# Stage 2: Runtime environment
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023-minimal as model_runner
 
-# set all the ENV vars needed for build
+# Set up runtime environment variables
 ENV CONDA_TARGET_ENV=osml_model_runner
-ENV CC="clang"
-ENV CXX="clang++"
-ENV ARCHFLAGS="-arch x86_64"
-ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/conda/lib/:/opt/conda/bin:/usr/include:/usr/local/"
-ENV PROJ_LIB=/opt/conda/share/proj
-
-# copy our conda env configuration for Python 3.10
-COPY environment-py310.yml environment.yml
-
-# create the conda env
-RUN conda env create
-
-# create /entry.sh which will be our new shell entry point
-# this performs actions to configure the environment
-# before starting a new shell (which inherits the env).
-# the exec is important as this allows signals to passpw
+ENV PATH=/opt/conda/bin:$PATH
+
+# Set working directory
+WORKDIR /home
+
+# Copy the conda environment from the build stage
+COPY --from=build /opt/conda /opt/conda
+RUN ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh
+
+# Copy the application from the build stage
+COPY --from=build /home/osml-model-runner /home/osml-model-runner
+
+# Create entrypoint script
 RUN     (echo '#!/bin/bash' \
     &&   echo '__conda_setup="$(/opt/conda/bin/conda shell.bash hook 2> /dev/null)"' \
     &&   echo 'eval "$__conda_setup"' \
@@ -51,29 +67,12 @@ RUN     (echo '#!/bin/bash' \
     &&   echo 'exec "$@"'\
         ) >> /entry.sh && chmod +x /entry.sh
 
-# tell the docker build process to use this for RUN.
-# the default shell on Linux is ["/bin/sh", "-c"], and on Windows is ["cmd", "/S", "/C"]
-SHELL ["/entry.sh", "/bin/bash", "-c"]
-
-# configure .bashrc to drop into a conda env and immediately activate our TARGET env
-RUN conda init && echo 'conda activate "${CONDA_TARGET_ENV:-base}"' >>  ~/.bashrc
-
-# copy our lcoal application source into the container
-COPY . osml-model-runner
-
-# install the model runner application from source
-RUN python3 -m pip install osml-model-runner/.
-
-# clean up the conda install
-RUN conda clean -afy
-
-# set up a health check at that port
-HEALTHCHECK NONE
+# Configure user and permissions
+RUN dnf install -y shadow-utils
+RUN adduser modelrunner && \
+    chown -R modelrunner:modelrunner ./
 
-# set up a user to run the container as and assume it
-RUN adduser modelrunner
-RUN chown -R modelrunner:modelrunner ./
 USER modelrunner
 
-# set the entry point script
+# Set entry point
 ENTRYPOINT ["/entry.sh", "/bin/bash", "-c", "python3 osml-model-runner/bin/oversightml-mr-entry-point.py"]
diff --git a/bin/oversightml-mr-entry-point.py b/bin/oversightml-mr-entry-point.py
@@ -9,15 +9,23 @@
 from typing import Optional
 
 from codeguru_profiler_agent import Profiler
+from pythonjsonlogger import jsonlogger
 
 from aws.osml.model_runner.app import ModelRunner
+from aws.osml.model_runner.common import ThreadingLocalContextFilter
 
 
 def handler_stop_signals(signal_num: int, frame: Optional[FrameType], model_runner: ModelRunner) -> None:
     model_runner.stop()
 
 
 def configure_logging(verbose: bool) -> None:
+    """
+    This function configures the Python logging module to use a JSON formatter with and thread local context
+    variables.
+
+    :param verbose: if true the logging level will be set to DEBUG, otherwise it will be set to INFO.
+    """
     logging_level = logging.INFO
     if verbose:
         logging_level = logging.DEBUG
@@ -27,7 +35,10 @@ def configure_logging(verbose: bool) -> None:
 
     ch = logging.StreamHandler()
     ch.setLevel(logging_level)
-    formatter = logging.Formatter("%(levelname)-8s %(message)s")
+    ch.addFilter(ThreadingLocalContextFilter(["job_id", "image_id"]))
+    formatter = jsonlogger.JsonFormatter(
+        fmt="%(levelname)s %(message)s %(job_id)s %(image_id)s", datefmt="%Y-%m-%dT%H:%M:%S"
+    )
     ch.setFormatter(formatter)
 
     root_logger.addHandler(ch)

diff --git a/environment-py310.yml b/environment-py310.yml
diff --git a/environment-py311.yml b/environment-py311.yml
@@ -0,0 +1,7 @@
+name: osml_model_runner
+channels:
+  - conda-forge
+dependencies:
+  - conda-forge::python=3.11.6
+  - conda-forge::gdal=3.8.3
+  - conda-forge::proj=9.3.1
diff --git a/environment.yml b/environment.yml
@@ -2,5 +2,5 @@ name: osml_model_runner
 channels:
   - conda-forge
 dependencies:
-  - conda-forge::gdal=3.7.0
-  - conda-forge::proj=9.2.1
+  - conda-forge::gdal=3.8.3
+  - conda-forge::proj=9.3.1
diff --git a/setup.cfg b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = osml-model-runner
-version = 1.3.0
+version = 1.4.0
 description = Application to run large scale imagery against AI/ML models
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -40,20 +40,21 @@ include_package_data = True
 
 install_requires =
     osml-imagery-toolkit>=1.2.0
-    numpy>=1.23.0
-    shapely>=1.8.5
-    aws-embedded-metrics==3.1.0
-    boto3==1.28.1
-    botocore==1.31.1
-    setuptools==68.0.0
-    cachetools==5.3.0
-    geojson>=3.0.0
+    numpy>=1.24.4
+    shapely>=2.0.2
+    aws-embedded-metrics==3.2.0
+    python-json-logger>=2.0.0
+    boto3==1.34.28
+    botocore==1.34.28
+    setuptools==69.0.3
+    cachetools==5.3.2
+    geojson>=3.1.0
     scikit-optimize>=0.9.0
-    pyproj>=3.6.0
+    pyproj>=3.6.1
     scipy==1.9.1;python_version<'3.11.0'
-    scipy==1.11.0;python_version>='3.11'
+    scipy==1.12.0;python_version>='3.11'
     argparse==1.4.0
-    dacite==1.8.0
+    dacite==1.8.1
     ensemble-boxes==1.0.9
     codeguru-profiler-agent==1.2.4
     defusedxml>=0.7.1
@@ -70,6 +71,6 @@ package_data =
 
 [options.extras_require]
 gdal =
-    gdal>=3.7.0
+    gdal>=3.8.3
 test =
     tox