diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml new file mode 100644 index 00000000..c717ec82 --- /dev/null +++ b/.github/workflows/lambda.yml @@ -0,0 +1,47 @@ +name: Lambda + +on: + push: + +jobs: + package: + name: Python Package + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + package: + - mailer + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + cache: pip + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt + + - name: Prepare Python env + run: | + python -m pip install -U pip setuptools wheel + + - name: Create build info + run: | + bash scripts/build-info.sh + + - name: Install dependencies + run: | + cd packages/dsw-${{ matrix.package }} + make build-lambda-package + + - name: Create artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.package }}-lambda.zip + path: packages/dsw-${{ matrix.package }}/${{ matrix.package }}-lambda.zip diff --git a/packages/dsw-command-queue/dsw/command_queue/command_queue.py b/packages/dsw-command-queue/dsw/command_queue/command_queue.py index 477a7622..1e410a7a 100644 --- a/packages/dsw-command-queue/dsw/command_queue/command_queue.py +++ b/packages/dsw-command-queue/dsw/command_queue/command_queue.py @@ -104,6 +104,18 @@ def run(self): LOG.info(f'Notifications received ({notifications})') LOG.debug('Exiting command queue') + @tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=RETRY_QUEUE_MULTIPLIER), + stop=tenacity.stop_after_attempt(RETRY_QUEUE_TRIES), + before=tenacity.before_log(LOG, logging.INFO), + after=tenacity.after_log(LOG, logging.INFO), + ) + def run_once(self): + LOG.info('Fetching the commands') + while self.fetch_and_process(): + pass + def accept_notification(self, payload: psycopg.Notify) -> bool: LOG.debug(f'Accepting notification from channel "{payload.channel}" ' f'(PID = {payload.pid}) {payload.payload}') diff --git a/packages/dsw-mailer/Makefile b/packages/dsw-mailer/Makefile index 906797c3..2bd63cff 100644 --- a/packages/dsw-mailer/Makefile +++ b/packages/dsw-mailer/Makefile @@ -17,3 +17,41 @@ docker-image-name: .PHONY: test test: @echo "No tests for this package" + +.PHONE: install-python-requirements +install-python-requirements: + python3.11 -m pip install --upgrade --target ./package -r $(FILE) + +.PHONE: install-dependency +install-dependency: + $(MAKE) install-python-requirements FILE=../$(MODULE)/requirements.txt + cp -R ../$(MODULE)/dsw ./package + +.PHONY: build-lambda-package +build-lambda-package: + $(MAKE) clean + + # 1. Install and copy DSW packages + mkdir package + $(MAKE) install-python-requirements FILE=../../requirements.txt + $(MAKE) install-dependency MODULE=dsw-command-queue + $(MAKE) install-dependency MODULE=dsw-config + $(MAKE) install-dependency MODULE=dsw-database + $(MAKE) install-dependency MODULE=dsw-models + $(MAKE) install-dependency MODULE=dsw-storage + $(MAKE) install-dependency MODULE=dsw-mailer + + # 2. Fix postgres library for Lambda + python3.11 -m pip install --upgrade --target ./package --platform manylinux2014_x86_64 --implementation cp --only-binary=:all: "psycopg[binary]" + cp package/psycopg_binary/_psycopg.cpython-311-x86_64-linux-gnu.so package/psycopg/_psycopg.so + cp package/psycopg_binary/_psycopg.cpython-311-x86_64-linux-gnu.so package/psycopg_binary/_psycopg.so + + # 3. Package the ZIP and clean + cd package && zip -r ../mailer-lambda.zip . + rm -rf package + +.PHONY: clean +clean: + rm -rf src/__pycache__ + rm -rf package + rm mailer-lambda.zip || true diff --git a/packages/dsw-mailer/dsw/mailer/__init__.py b/packages/dsw-mailer/dsw/mailer/__init__.py index cfabb236..76f5e4f1 100644 --- a/packages/dsw-mailer/dsw/mailer/__init__.py +++ b/packages/dsw-mailer/dsw/mailer/__init__.py @@ -1,3 +1,4 @@ from .cli import main +from .lambda_handler import lambda_handler -__all__ = ['main'] +__all__ = ['main', 'lambda_handler'] diff --git a/packages/dsw-mailer/dsw/mailer/lambda_handler.py b/packages/dsw-mailer/dsw/mailer/lambda_handler.py new file mode 100644 index 00000000..67f5ca31 --- /dev/null +++ b/packages/dsw-mailer/dsw/mailer/lambda_handler.py @@ -0,0 +1,12 @@ +import pathlib + +from dsw.mailer.cli import validate_config +from dsw.mailer.mailer import Mailer + + +def lambda_handler(event, context): + with open("/var/task/application.yml", "r") as config_file: + config = validate_config(None, None, config_file) + config.log.apply() + mailer = Mailer(config, pathlib.Path("/var/task/templates")) + mailer.run_once() diff --git a/packages/dsw-mailer/dsw/mailer/mailer.py b/packages/dsw-mailer/dsw/mailer/mailer.py index a09555a1..1ce578f8 100644 --- a/packages/dsw-mailer/dsw/mailer/mailer.py +++ b/packages/dsw-mailer/dsw/mailer/mailer.py @@ -81,6 +81,21 @@ def run(self): ) queue.run() + def run_once(self): + Context.get().app.db.connect() + # prepare + self._update_component_info() + # work in queue + LOG.info('Preparing command queue') + queue = CommandQueue( + worker=self, + db=Context.get().app.db, + channel=CMD_CHANNEL, + component=CMD_COMPONENT, + timeout=Context.get().app.cfg.db.queue_timout, + ) + queue.run_once() + def work(self, cmd: PersistentCommand): # update Sentry info SentryReporter.set_context('template', '-')