release: v1.2.0

Release/v1.2.0

.github/workflows/docker-build.yml

@@ -1,21 +1,14 @@
 name: Build Docker Container
 
 on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
+  workflow_call:
 
 jobs:
-
   build:
-
     runs-on: ubuntu-latest
-
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         lfs: 'true'
-        ssh-key: ${{ secrets.git_ssh_key }}
     - name: Build the Docker image
       run: docker build . --file Dockerfile --tag osml-model-runner:$(date +%s)

.github/workflows/docker-publish-dockerhub.yml

@@ -1,43 +1,33 @@
-name: Publish DockerHub Container
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
+name: Publish OSML Model Runner DockerHub Container
 
-# GitHub recommends pinning actions to a commit SHA.
-# To get a newer version, you will need to update the SHA.
-# You can also reference a tag or branch, but the action may change without warning.
-
-# Configures this workflow to run every time a change is pushed to the branch called `main`.
 on:
-  push:
-    branches: ['main']
+  workflow_call:
+
+env:
+  REGISTRY: awsosml
+  IMAGE_NAME: ${{ github.event.repository.name }}
 
 jobs:
   push_to_registry:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: 'true'
-          ssh-key: ${{ secrets.git_ssh_key }}
-
       - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@v5
         with:
-          images: awsosml/osml-model-runner
-
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - name: Publish Container to Dockerhub
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile

.github/workflows/docker-publish-github.yml

@@ -1,45 +1,36 @@
-name: Publish GitHub Container
+name: Publish OSML Model Runner GitHub Container
 
-# Configures this workflow to run every time a change is pushed to the branch called `main`.
 on:
-  push:
-    branches: ['main']
+  workflow_call:
 
-# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
-# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
 jobs:
   build-and-push-image:
     runs-on: ubuntu-latest
-    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
     permissions:
       contents: read
       packages: write
-      #
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
-      # Uses the `docker/login-action` action to log in to the Container registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+        uses: actions/checkout@v4
+        with:
+          lfs: 'true'
       - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
-      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
-      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
       - name: Push to Github
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true

.github/workflows/documentation.yaml → .github/workflows/documentation-publish.yml

@@ -1,26 +1,31 @@
-name: Docs
-on: [push, pull_request, workflow_dispatch]
+name: Generate and Publish Documentation for OSML Model Runner
+
+on:
+  workflow_call:
+
 permissions:
     contents: write
+
 jobs:
   docs:
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v3
       - name: Install dependencies
         run: |
           pip install sphinx
           python -m pip install sphinx-autoapi
           python -m pip install sphinx_rtd_theme
+          python -m pip install tox tox-gh-actions
       - name: Sphinx build
         run: |
-          sphinx-build doc _build
+          tox -e docs
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3.9.3
-        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
         with:
           publish_branch: gh-pages
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: _build/
+          publish_dir: .tox/docs/tmp/html/
           force_orphan: true

.github/workflows/notify-guidance-repo.yml

@@ -0,0 +1,18 @@
+name: Notify Guidance Overhead Imagery Repo
+
+on:
+  workflow_call:
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Short Sha
+      uses: benjlevesque/short-sha@v2.2
+    - name: Invoke Guidance OSML Submodule Workflow
+      uses: benc-uk/workflow-dispatch@v1.2.2
+      with:
+        repo: aws-solutions-library-samples/guidance-for-overhead-imagery-inference-on-aws
+        workflow: osml_update_submodules.yml
+        token: ${{ secrets.GUIDANCE_OSML_SUBMODULES_ACTION_10_24 }}
+        inputs: '{ "DISPATCH_REPO_NAME" : "${{ github.event.repository.name }}", "DISPATCH_REPO_SHA": "${{ env.SHA }}" }'

.github/workflows/osml-model-runner-build.yml

@@ -0,0 +1,13 @@
+name: "OSML Model Runner Build Workflow"
+
+on:
+  pull_request:
+    branches: ["main", "dev"]
+
+jobs:
+  Build_Validate_Tox:
+    uses: ./.github/workflows/python-tox.yml
+    secrets: inherit
+  Build_Docker_Container:
+    uses: ./.github/workflows/docker-build.yml
+    secrets: inherit

.github/workflows/osml-model-runner-publish.yml

@@ -0,0 +1,33 @@
+name: "OSML Model Runner Build and Publish Workflow"
+
+on:
+  push:
+    branches: ["main", "dev"]
+
+jobs:
+  Build_Validate_Tox:
+    uses: ./.github/workflows/python-tox.yml
+    secrets: inherit
+  Build_Docker_Container:
+    uses: ./.github/workflows/docker-build.yml
+    secrets: inherit
+  Publish_Python:
+    needs: [Build_Validate_Tox, Build_Docker_Container]
+    uses: ./.github/workflows/python-publish.yml
+    secrets: inherit
+  Publish_Docker_Dockerhub:
+    needs: [Build_Validate_Tox, Build_Docker_Container]
+    uses: ./.github/workflows/docker-publish-dockerhub.yml
+    secrets: inherit
+  Publish_Docker_Github:
+    needs: [Build_Validate_Tox, Build_Docker_Container]
+    uses: ./.github/workflows/docker-publish-github.yml
+    secrets: inherit
+  Publish_Documentation:
+    needs: [Publish_Python, Publish_Docker_Github, Publish_Docker_Dockerhub]
+    uses: ./.github/workflows/documentation-publish.yml
+    secrets: inherit
+  Notify_Guidance_Repo:
+    needs: [Publish_Docker_Github, Publish_Docker_Dockerhub]
+    uses: ./.github/workflows/notify-guidance-repo.yml
+    secrets: inherit

.github/workflows/python-publish.yml

@@ -1,14 +1,10 @@
 # This workflow will upload a Python Package using Twine when a release is created
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
 
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
 name: Publish Python Package
 
 on:
+  workflow_call:
   release:
     types: [published]
 
@@ -17,11 +13,10 @@ permissions:
 
 jobs:
   deploy:
-
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
     runs-on: ubuntu-latest
-
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v3
       with:
@@ -33,7 +28,7 @@ jobs:
     - name: Build package
       run: python -m build
     - name: Publish package
-      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      uses: pypa/gh-action-pypi-publish@v1.8.10
       with:
         user: __token__
         password: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/python-tox.yml

@@ -1,27 +1,18 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
-
 name: Build/Validation with Tox
 
 on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
+  workflow_call:
 
 permissions:
   contents: read
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
-
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         lfs: 'true'
-        ssh-key: ${{ secrets.git_ssh_key }}
     - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:

scripts/run_container.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Copyright 2023 Amazon.com, Inc. or its affiliates.
+#
+
+# This is a utility to run a local model runner container with an ENV imported from ECS.
+# $1 = PATTERN = the pattern contained in the ECS task definition you want to import from
+# $2 = CONTAINER_NAME = the name of the docker image you want to run
+# $3 = AWS_REGION = region the ecs task definition is contained in
+
+echo "    ____                    _                __                     __";
+echo "   / __ \__  ______  ____  (_)___  ____ _   / /   ____  _________ _/ /";
+echo "  / /_/ / / / / __ \/ __ \/ / __ \/ __  /  / /   / __ \/ ___/ __  / /";
+echo " / _, _/ /_/ / / / / / / / / / / / /_/ /  / /___/ /_/ / /__/ /_/ / / ";
+echo "/_/ |_|\__,_/_/ /_/_/ /_/_/_/ /_/\__, /  /_____/\____/\___/\__,_/_/";
+echo "    __  _______     ______      /____/__        _";
+echo "   /  |/  / __ \   / ____/___  ____  / /_____ _(_)___  ___  _____";
+echo "  / /|_/ / /_/ /  / /   / __ \/ __ \/ __/ __  / / __ \/ _ \/ ___/";
+echo " / /  / / _, _/  / /___/ /_/ / / / / /_/ /_/ / / / / /  __/ /";
+echo "/_/  /_/_/ |_|   \____/\____/_/ /_/\__/\__,_/_/_/ /_/\___/_/ ";
+
+
+# Inputs
+PATTERN="${1:-"MRDataplane"}"
+IMAGE_NAME=PATTERN="${2:-"osml-model-runner:local"}"
+AWS_REGION="${3:-"us-west-2"}"
+
+# Get the latest task definition ARN based on a string pattern
+LATEST_TASK_DEFINITION_ARN=$(aws ecs list-task-definitions --region $AWS_REGION --sort DESC | jq -r ".taskDefinitionArns[] | select(. | contains(\"$PATTERN\"))" | head -n 1)
+
+if [ -z "$LATEST_TASK_DEFINITION_ARN" ]; then
+    echo "No task definition found with pattern: $PATTERN"
+    exit 1
+fi
+
+echo "Latest task definition ARN with pattern $PATTERN is: $LATEST_TASK_DEFINITION_ARN"
+
+# Extract environment variables from ECS task definition using AWS CLI and jq
+ENV_VARS=$(aws ecs describe-task-definition --region $AWS_REGION --task-definition "$LATEST_TASK_DEFINITION_ARN" | jq -r '.taskDefinition.containerDefinitions[0].environment[] | "-e " + .name + "=\"" + .value + "\"" ' | tr '\n' ' ')
+
+# If aws cli or jq command fails
+if [ $? -ne 0 ]; then
+    echo "Failed to fetch environment variables from ECS task definition."
+    exit 1
+fi
+
+echo "Running with env variables: $ENV_VARS"
+
+# Give the container permissions on aws credentials
+echo "Warning: Giving 777 permissions on ~/.aws/credentials - revert to desired permissions if needed"
+chmod 777 ~/.aws/credentials
+
+# Note: The `-d` flag runs the container in detached mode.
+# Add it if you want to run the container in the background.
+# Run Docker container with environment variables and mount AWS credentials
+DOCKER_CMD="docker run \
+  -p 8080:8080 \
+  -v ~/.aws/credentials:/home/modelrunner/.aws/credentials:rw \
+  $ENV_VARS \
+  $IMAGE_NAME"
+
+# Run Docker container with environment variables
+eval "$DOCKER_CMD"

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = osml-model-runner
-version = 1.1.0
+version = 1.2.0
 description = Application to run large scale imagery against AI/ML models
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -57,7 +57,7 @@ install_requires =
     ensemble-boxes==1.0.9
     codeguru-profiler-agent==1.2.4
     defusedxml>=0.7.1
-
+    requests==2.31.0
 
 [options.packages.find]
 where = src