diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b28a2900..e903d975 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -176,6 +176,7 @@ jobs: with: images: | ${{ env.PUBLIC_IMAGE_PREFIX }}/${{ steps.docker-image-name.outputs.NAME }} + ${{ secrets.DOCKER_HUB_USERNAME }}/${{ steps.docker-image-name.outputs.NAME }} tags: | type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} type=semver,pattern={{version}} diff --git a/packages/dsw-command-queue/CHANGELOG.md b/packages/dsw-command-queue/CHANGELOG.md index 3c69959f..1e5c9a92 100644 --- a/packages/dsw-command-queue/CHANGELOG.md +++ b/packages/dsw-command-queue/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.9.0] + +Released for version consistency with other DSW tools. + ## [4.8.1] Released for version consistency with other DSW tools. @@ -221,3 +225,4 @@ Released for version consistency with other DSW tools. [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml index efa59fda..a3706ca2 100644 --- a/packages/dsw-command-queue/pyproject.toml +++ b/packages/dsw-command-queue/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-command-queue' -version = "4.8.1" +version = "4.9.0" description = 'Library for working with command queue and persistent commands' readme = 'README.md' keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing'] @@ -25,7 +25,7 @@ classifiers = [ requires-python = '>=3.10, <4' dependencies = [ # DSW - "dsw-database==4.8.1", + "dsw-database==4.9.0", ] [project.urls] diff --git a/packages/dsw-command-queue/requirements.txt b/packages/dsw-command-queue/requirements.txt index d29d54fd..c59318e6 100644 --- a/packages/dsw-command-queue/requirements.txt +++ b/packages/dsw-command-queue/requirements.txt @@ -1,6 +1,6 @@ -psycopg==3.1.19 -psycopg-binary==3.1.19 +psycopg==3.2.1 +psycopg-binary==3.2.1 PyYAML==6.0.1 -tenacity==8.4.1 +tenacity==8.5.0 typing_extensions==4.12.2 tzdata==2024.1 diff --git a/packages/dsw-config/CHANGELOG.md b/packages/dsw-config/CHANGELOG.md index 13aea3f4..f969a8ef 100644 --- a/packages/dsw-config/CHANGELOG.md +++ b/packages/dsw-config/CHANGELOG.md @@ -8,6 +8,16 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.9.0] + +### Added + +- Included AWS-specific configuration that `engine-backend` uses + +### Changed + +- Moved `mail` configuration to `dsw-mailer` package + ## [4.8.1] Released for version consistency with other DSW tools. @@ -227,3 +237,4 @@ Released for version consistency with other DSW tools. [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-config/dsw/config/keys.py b/packages/dsw-config/dsw/config/keys.py index 615484a7..3a9cccf6 100644 --- a/packages/dsw-config/dsw/config/keys.py +++ b/packages/dsw-config/dsw/config/keys.py @@ -232,99 +232,30 @@ class _S3Keys(ConfigKeysContainer): ) -class _MailKeys(ConfigKeysContainer): - enabled = ConfigKey( - yaml_path=['mail', 'enabled'], - var_names=['MAIL_ENABLED'], - default=True, - cast=cast_bool, - ) - name = ConfigKey( - yaml_path=['mail', 'name'], - var_names=['MAIL_NAME'], - default='', - cast=cast_str, - ) - email = ConfigKey( - yaml_path=['mail', 'email'], - var_names=['MAIL_EMAIL'], - default='', - cast=cast_str, - ) - host = ConfigKey( - yaml_path=['mail', 'host'], - var_names=['MAIL_HOST'], - default='', - cast=cast_str, - ) - port = ConfigKey( - yaml_path=['mail', 'port'], - var_names=['MAIL_PORT'], - cast=cast_str, - ) - ssl = ConfigKey( - yaml_path=['mail', 'ssl'], - var_names=[], - cast=cast_optional_str, - ) - security = ConfigKey( - yaml_path=['mail', 'security'], - var_names=['MAIL_SECURITY'], +class _AWSKeys(ConfigKeysContainer): + access_key_id = ConfigKey( + yaml_path=['aws', 'awsAccessKeyId'], + var_names=['AWS_AWS_ACCESS_KEY_ID'], cast=cast_optional_str, ) - auth_enabled = ConfigKey( - yaml_path=['mail', 'authEnabled'], - var_names=[], - cast=cast_optional_bool, - ) - username = ConfigKey( - yaml_path=['mail', 'username'], - var_names=['MAIL_USERNAME'], + secret_access_key = ConfigKey( + yaml_path=['aws', 'awsSecretAccessKey'], + var_names=['AWS_AWS_SECRET_ACCESS_KEY'], cast=cast_optional_str, ) - password = ConfigKey( - yaml_path=['mail', 'password'], - var_names=['MAIL_PASSWORD'], - cast=cast_optional_str, - ) - rate_limit_window = ConfigKey( - yaml_path=['mail', 'rateLimit', 'window'], - var_names=['MAIL_RATE_LIMIT_WINDOW'], - default=0, - cast=cast_int, - ) - rate_limit_count = ConfigKey( - yaml_path=['mail', 'rateLimit', 'count'], - var_names=['MAIL_RATE_LIMIT_COUNT'], - default=0, - cast=cast_int, - ) - timeout = ConfigKey( - yaml_path=['mail', 'timeout'], - var_names=['MAIL_TIMEOUT'], - default=10, - cast=cast_int, - ) - dkim_selector = ConfigKey( - yaml_path=['mail', 'dkim', 'selector'], - var_names=['MAIL_DKIM_SELECTOR'], - default=None, - cast=cast_optional_str, - ) - dkim_privkey_file = ConfigKey( - yaml_path=['mail', 'dkim', 'privkey_file'], - var_names=['MAIL_DKIM_PRIVKEY_FILE'], - default=None, + region = ConfigKey( + yaml_path=['aws', 'awsRegion'], + var_names=['AWS_AWS_REGION'], cast=cast_optional_str, ) class ConfigKeys(ConfigKeysContainer): + aws = _AWSKeys cloud = _CloudKeys database = _DatabaseKeys general = _GeneralKeys logging = _LoggingKeys - mail = _MailKeys s3 = _S3Keys sentry = _SentryKeys diff --git a/packages/dsw-config/dsw/config/model.py b/packages/dsw-config/dsw/config/model.py index d6cd0601..e5e12095 100644 --- a/packages/dsw-config/dsw/config/model.py +++ b/packages/dsw-config/dsw/config/model.py @@ -1,4 +1,3 @@ -import pathlib from typing import Optional from .logging import prepare_logging, LOG_FILTER @@ -74,93 +73,20 @@ def set_logging_extra(key: str, value: str): LOG_FILTER.set_extra(key, value) -class CloudConfig(ConfigModel): - - def __init__(self, multi_tenant: bool): - self.multi_tenant = multi_tenant - - -class MailConfig(ConfigModel): - - def __init__(self, enabled: bool, ssl: Optional[bool], name: str, email: str, - host: str, port: Optional[int], security: Optional[str], - auth_enabled: Optional[bool], username: Optional[str], - password: Optional[str], rate_limit_window: int, - rate_limit_count: int, timeout: int, - dkim_selector: Optional[str] = None, - dkim_privkey_file: Optional[str] = None): - self.enabled = enabled - self.name = name - self.email = email - self.host = host - self.security = 'plain' - if security is not None: - self.security = security.lower() - elif ssl is not None: - self.security = 'ssl' if ssl else 'plain' - self.port = port or self._default_port() - self.auth = auth_enabled - if self.auth is None: - self.auth = username is not None and password is not None - self.username = username - self.password = password - self.rate_limit_window = rate_limit_window - self.rate_limit_count = rate_limit_count - self.timeout = timeout - self.dkim_selector = dkim_selector - self.dkim_privkey_file = dkim_privkey_file - self.dkim_privkey = b'' - - def load_dkim_privkey(self): - if self.dkim_privkey_file is not None: - self.dkim_privkey = pathlib.Path(self.dkim_privkey_file).read_bytes() - self.dkim_privkey = self.dkim_privkey.replace(b'\r\n', b'\n') - - @property - def use_dkim(self): - return self.dkim_selector is not None and len(self.dkim_privkey) > 0 - - @property - def login_user(self) -> str: - return self.username or '' - - @property - def login_password(self) -> str: - return self.password or '' +class AWSConfig(ConfigModel): - @property - def is_plain(self): - return self.security == 'plain' - - @property - def is_ssl(self): - return self.security == 'ssl' + def __init__(self, access_key_id: Optional[str], secret_access_key: Optional[str], + region: Optional[str]): + self.access_key_id = access_key_id + self.secret_access_key = secret_access_key + self.region = region @property - def is_tls(self): - return self.security == 'starttls' or self.security == 'tls' + def has_credentials(self) -> bool: + return self.access_key_id is not None and self.secret_access_key is not None - def _default_port(self) -> int: - if self.is_plain: - return 25 - if self.is_ssl: - return 465 - return 587 - def has_credentials(self) -> bool: - return self.username is not None and self.password is not None +class CloudConfig(ConfigModel): - def __str__(self): - return f'MailConfig\n' \ - f'- enabled = {self.enabled}\n' \ - f'- name = {self.name}\n' \ - f'- email = {self.email}\n' \ - f'- host = {self.host}\n' \ - f'- port = {self.port}\n' \ - f'- security = {self.security}\n' \ - f'- auth = {self.auth}\n' \ - f'- rate_limit_window = {self.rate_limit_window}\n' \ - f'- rate_limit_count = {self.rate_limit_count}\n' \ - f'- timeout = {self.timeout}\n' \ - f'- dkim_selector = {self.dkim_selector}\n' \ - f'- dkim_privkey_file = {self.dkim_privkey_file}\n' + def __init__(self, multi_tenant: bool): + self.multi_tenant = multi_tenant diff --git a/packages/dsw-config/dsw/config/parser.py b/packages/dsw-config/dsw/config/parser.py index 92569c42..a3382422 100644 --- a/packages/dsw-config/dsw/config/parser.py +++ b/packages/dsw-config/dsw/config/parser.py @@ -5,7 +5,7 @@ from .keys import ConfigKey, ConfigKeys from .model import GeneralConfig, SentryConfig, S3Config, \ - DatabaseConfig, LoggingConfig, CloudConfig, MailConfig + DatabaseConfig, LoggingConfig, CloudConfig, AWSConfig class MissingConfigurationError(Exception): @@ -129,21 +129,9 @@ def general(self) -> GeneralConfig: ) @property - def mail(self): - return MailConfig( - enabled=self.get(self.keys.mail.enabled), - name=self.get(self.keys.mail.name), - email=self.get(self.keys.mail.email), - host=self.get(self.keys.mail.host), - ssl=self.get(self.keys.mail.ssl), - port=self.get(self.keys.mail.port), - security=self.get(self.keys.mail.security), - auth_enabled=self.get(self.keys.mail.auth_enabled), - username=self.get(self.keys.mail.username), - password=self.get(self.keys.mail.password), - rate_limit_window=int(self.get(self.keys.mail.rate_limit_window)), - rate_limit_count=int(self.get(self.keys.mail.rate_limit_count)), - timeout=int(self.get(self.keys.mail.timeout)), - dkim_selector=self.get(self.keys.mail.dkim_selector), - dkim_privkey_file=self.get(self.keys.mail.dkim_privkey_file), + def aws(self) -> AWSConfig: + return AWSConfig( + access_key_id=self.get(self.keys.aws.access_key_id), + secret_access_key=self.get(self.keys.aws.secret_access_key), + region=self.get(self.keys.aws.region), ) diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index 14d18055..a0f80597 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-config' -version = "4.8.1" +version = "4.9.0" description = 'Library for DSW config manipulation' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-config/requirements.txt b/packages/dsw-config/requirements.txt index 320d9a0a..e47e6991 100644 --- a/packages/dsw-config/requirements.txt +++ b/packages/dsw-config/requirements.txt @@ -1,4 +1,4 @@ -certifi==2024.6.2 +certifi==2024.7.4 PyYAML==6.0.1 -sentry-sdk==2.6.0 +sentry-sdk==2.8.0 urllib3==2.2.2 diff --git a/packages/dsw-data-seeder/CHANGELOG.md b/packages/dsw-data-seeder/CHANGELOG.md index f91d07cd..c92d61ef 100644 --- a/packages/dsw-data-seeder/CHANGELOG.md +++ b/packages/dsw-data-seeder/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.9.0] + +Released for version consistency with other DSW tools. + ## [4.8.1] Released for version consistency with other DSW tools. @@ -277,3 +281,4 @@ Released for version consistency with other DSW tools. [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-data-seeder/dsw/data_seeder/consts.py b/packages/dsw-data-seeder/dsw/data_seeder/consts.py index de7c3b7d..10b7f2dd 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/consts.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/consts.py @@ -6,7 +6,7 @@ DEFAULT_PLACEHOLDER = '<<|TENANT-ID|>>' NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'dsw-data-seeder' -VERSION = '4.8.1' +VERSION = '4.9.0' VAR_APP_CONFIG_PATH = 'APPLICATION_CONFIG_PATH' VAR_WORKDIR_PATH = 'WORKDIR_PATH' diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml index acf4d8b9..6be59e82 100644 --- a/packages/dsw-data-seeder/pyproject.toml +++ b/packages/dsw-data-seeder/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-data-seeder' -version = "4.8.1" +version = "4.9.0" description = 'Worker for seeding DSW data' readme = 'README.md' keywords = ['data', 'database', 'seed', 'storage'] @@ -29,10 +29,10 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - "dsw-command-queue==4.8.1", - "dsw-config==4.8.1", - "dsw-database==4.8.1", - "dsw-storage==4.8.1", + "dsw-command-queue==4.9.0", + "dsw-config==4.9.0", + "dsw-database==4.9.0", + "dsw-storage==4.9.0", ] [project.urls] diff --git a/packages/dsw-data-seeder/requirements.txt b/packages/dsw-data-seeder/requirements.txt index 8c5198ff..4b0d0e03 100644 --- a/packages/dsw-data-seeder/requirements.txt +++ b/packages/dsw-data-seeder/requirements.txt @@ -1,12 +1,12 @@ -certifi==2024.6.2 +certifi==2024.7.4 click==8.1.7 minio==7.2.7 -psycopg==3.1.19 -psycopg-binary==3.1.19 +psycopg==3.2.1 +psycopg-binary==3.2.1 python-dateutil==2.9.0 PyYAML==6.0.1 -sentry-sdk==2.6.0 +sentry-sdk==2.8.0 six==1.16.0 -tenacity==8.4.1 +tenacity==8.5.0 typing_extensions==4.12.2 urllib3==2.2.2 diff --git a/packages/dsw-database/CHANGELOG.md b/packages/dsw-database/CHANGELOG.md index 2101382f..806f5550 100644 --- a/packages/dsw-database/CHANGELOG.md +++ b/packages/dsw-database/CHANGELOG.md @@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.9.0] + +### Changed + +- Updated `instance_mail_config` to support Amazon SES configuration + ## [4.8.1] ### Changed @@ -242,3 +248,4 @@ Released for version consistency with other DSW tools. [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-database/dsw/database/database.py b/packages/dsw-database/dsw/database/database.py index 2d31a96c..c834a6e4 100644 --- a/packages/dsw-database/dsw/database/database.py +++ b/packages/dsw-database/dsw/database/database.py @@ -1,6 +1,7 @@ import datetime import logging import psycopg +import psycopg.conninfo import psycopg.rows import psycopg.types.json import tenacity @@ -494,9 +495,11 @@ def __init__(self, name: str, dsn: str, timeout=30000, autocommit=False): ) def _connect_db(self): LOG.info(f'Creating connection to PostgreSQL database "{self.name}"') - connection = psycopg.connect(conninfo=self.dsn, autocommit=self.autocommit) - if connection is None: - raise RuntimeError('Failed to init DB connection') + try: + connection = psycopg.connect(conninfo=self.dsn, autocommit=self.autocommit) # type: psycopg.Connection + except Exception as e: + LOG.error(f'Failed to connect to PostgreSQL database "{self.name}": {str(e)}') + raise e # test connection cursor = connection.cursor() cursor.execute(query='SELECT 1;') diff --git a/packages/dsw-database/dsw/database/model.py b/packages/dsw-database/dsw/database/model.py index 641cd2ce..2461ee80 100644 --- a/packages/dsw-database/dsw/database/model.py +++ b/packages/dsw-database/dsw/database/model.py @@ -389,13 +389,17 @@ class DBInstanceConfigMail: uuid: str enabled: bool + provider: str sender_name: Optional[str] sender_email: Optional[str] - host: str - port: Optional[int] - security: Optional[str] - username: Optional[str] - password: Optional[str] + smtp_host: Optional[str] + smtp_port: Optional[int] + smtp_security: Optional[str] + smtp_username: Optional[str] + smtp_password: Optional[str] + aws_access_key_id: Optional[str] + aws_secret_access_key: Optional[str] + aws_region: Optional[str] rate_limit_window: Optional[int] rate_limit_count: Optional[int] timeout: Optional[int] @@ -405,13 +409,17 @@ def from_dict_row(data: dict): return DBInstanceConfigMail( uuid=str(data['uuid']), enabled=data['enabled'], + provider=data['provider'], sender_name=data['sender_name'], sender_email=data['sender_email'], - host=data['host'], - port=data['port'], - security=data['security'], - username=data['username'], - password=data['password'], + smtp_host=data['smtp_host'], + smtp_port=data['smtp_port'], + smtp_security=data['smtp_security'], + smtp_username=data['smtp_username'], + smtp_password=data['smtp_password'], + aws_access_key_id=data['aws_access_key_id'], + aws_secret_access_key=data['aws_secret_access_key'], + aws_region=data['aws_region'], rate_limit_window=data['rate_limit_window'], rate_limit_count=data['rate_limit_count'], timeout=data['timeout'], diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml index 72d895a3..b1b75df8 100644 --- a/packages/dsw-database/pyproject.toml +++ b/packages/dsw-database/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-database' -version = "4.8.1" +version = "4.9.0" description = 'Library for managing DSW database' readme = 'README.md' keywords = ['dsw', 'database'] @@ -26,7 +26,7 @@ dependencies = [ 'psycopg[binary]', 'tenacity', # DSW - "dsw-config==4.8.1", + "dsw-config==4.9.0", ] [project.urls] diff --git a/packages/dsw-database/requirements.txt b/packages/dsw-database/requirements.txt index d29d54fd..c59318e6 100644 --- a/packages/dsw-database/requirements.txt +++ b/packages/dsw-database/requirements.txt @@ -1,6 +1,6 @@ -psycopg==3.1.19 -psycopg-binary==3.1.19 +psycopg==3.2.1 +psycopg-binary==3.2.1 PyYAML==6.0.1 -tenacity==8.4.1 +tenacity==8.5.0 typing_extensions==4.12.2 tzdata==2024.1 diff --git a/packages/dsw-document-worker/CHANGELOG.md b/packages/dsw-document-worker/CHANGELOG.md index fb162006..652a8a44 100644 --- a/packages/dsw-document-worker/CHANGELOG.md +++ b/packages/dsw-document-worker/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.9.0] + +Released for version consistency with other DSW tools. + ## [4.8.1] ### Changed @@ -293,3 +297,4 @@ Released for version consistency with other DSW tools. [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-document-worker/dsw/document_worker/consts.py b/packages/dsw-document-worker/dsw/document_worker/consts.py index 81624419..2c237b2a 100644 --- a/packages/dsw-document-worker/dsw/document_worker/consts.py +++ b/packages/dsw-document-worker/dsw/document_worker/consts.py @@ -6,7 +6,7 @@ EXIT_SUCCESS = 0 NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'docworker' -VERSION = '4.8.1' +VERSION = '4.9.0' VAR_APP_CONFIG_PATH = 'APPLICATION_CONFIG_PATH' VAR_WORKDIR_PATH = 'WORKDIR_PATH' diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml index 9401149d..229c28b0 100644 --- a/packages/dsw-document-worker/pyproject.toml +++ b/packages/dsw-document-worker/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-document-worker' -version = "4.8.1" +version = "4.9.0" description = 'Worker for assembling and transforming documents' readme = 'README.md' keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker'] @@ -38,10 +38,10 @@ dependencies = [ 'weasyprint', 'XlsxWriter', # DSW - "dsw-command-queue==4.8.1", - "dsw-config==4.8.1", - "dsw-database==4.8.1", - "dsw-storage==4.8.1", + "dsw-command-queue==4.9.0", + "dsw-config==4.9.0", + "dsw-database==4.9.0", + "dsw-storage==4.9.0", ] [project.urls] diff --git a/packages/dsw-document-worker/requirements.txt b/packages/dsw-document-worker/requirements.txt index 33ba8f0f..6410374e 100644 --- a/packages/dsw-document-worker/requirements.txt +++ b/packages/dsw-document-worker/requirements.txt @@ -1,10 +1,10 @@ Brotli==1.1.0 -certifi==2024.6.2 +certifi==2024.7.4 cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 cssselect2==0.7.0 -fonttools==4.53.0 +fonttools==4.53.1 html5lib==1.1 idna==3.7 isodate==0.6.1 @@ -14,9 +14,9 @@ MarkupSafe==2.1.5 minio==7.2.7 panflute==2.3.1 pathvalidate==3.2.0 -Pillow==10.3.0 -psycopg==3.1.19 -psycopg-binary==3.1.19 +Pillow==10.4.0 +psycopg==3.2.1 +psycopg-binary==3.2.1 pycparser==2.22 pydyf==0.10.0 pyparsing==3.1.2 @@ -27,9 +27,9 @@ PyYAML==6.0.1 rdflib==7.0.0 rdflib-jsonld==0.6.2 requests==2.32.3 -sentry-sdk==2.6.0 +sentry-sdk==2.8.0 six==1.16.0 -tenacity==8.4.1 +tenacity==8.5.0 text-unidecode==1.3 tinycss2==1.3.0 typing_extensions==4.12.2 diff --git a/packages/dsw-mailer/CHANGELOG.md b/packages/dsw-mailer/CHANGELOG.md index 64ce6f8a..a06ea8f1 100644 --- a/packages/dsw-mailer/CHANGELOG.md +++ b/packages/dsw-mailer/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.9.0] + +Released for version consistency with other DSW tools. + ## [4.8.1] ### Changed @@ -253,3 +257,4 @@ Released for version consistency with other DSW tools. [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-mailer/config.example.yml b/packages/dsw-mailer/config.example.yml index 0a164638..766e90b6 100644 --- a/packages/dsw-mailer/config.example.yml +++ b/packages/dsw-mailer/config.example.yml @@ -1,9 +1,6 @@ database: connectionString: postgresql://postgres:postgres@postgres:5432/engine-wizard - stripeSize: 1 # used by SE only connectionTimeout: 30000 - maxConnections: 10 # used by SE only - queueTimeout: 500 # used by DW only s3: url: http://minio:9000 @@ -16,29 +13,43 @@ logging: globalLevel: WARNING mail: + # enabled: if false = no emails are sent (mailer will not send it, dry run) + enabled: true + # from (name, email): default sender email address and name name: email: - host: - # port: default (empty) = based on security 25/465/587 - port: - # security: plain | ssl | starttls - security: starttls - authEnabled: False - username: '' - password: '' - # legacy "ssl" flag (if security is missing) - # - true = security: ssl - # - false = security: plain - ssl: true - # legacy (if false, no emails are sent - dry-run) - enabled: true + # provider: smtp | amazonSes | none + provider: smtp + smtp: + host: + # port: default (empty) = based on security 25/465/587 + port: + # security: plain | ssl | starttls | tls + security: starttls + username: '' + password: '' + # SMTP connection timeout (default: 5 seconds) + timeout: 5 + amazonSes: + accessKeyId: + secretAccessKey: + region: rateLimit: # time windows in seconds, 0=disabled rate limit window: 300 # max number of messages within the window count: 10 - # SMTP connection timeout (default: 5 seconds) - timeout: 5 + dkim: + # DKIM key selector + selector: + # DKIM private key file (path) + privkeyFile: + +# AWS Configuration used by server (fallback if not provided for mail) +#aws: +# awsAccessKeyId: +# awsSecretAccessKey: +# awsRegion: #sentry: # enabled: diff --git a/packages/dsw-mailer/dsw/mailer/cli.py b/packages/dsw-mailer/dsw/mailer/cli.py index 10fb7e8f..52690b8a 100644 --- a/packages/dsw-mailer/dsw/mailer/cli.py +++ b/packages/dsw-mailer/dsw/mailer/cli.py @@ -2,7 +2,7 @@ import json import pathlib -from typing import IO, Optional +from typing import IO from dsw.config.parser import MissingConfigurationError @@ -32,7 +32,7 @@ def load_config_str(config_str: str) -> MailerConfig: return config -def validate_config(ctx, param, value: Optional[IO]): +def validate_config(ctx, param, value: IO | None): content = '' if value is not None: content = value.read() @@ -69,10 +69,13 @@ def cli(ctx, config: MailerConfig, workdir: str): @click.pass_context @click.argument('msg-request', type=click.File('r', encoding=DEFAULT_ENCODING), callback=extract_message_request) -def send(ctx, msg_request: MessageRequest): +@click.option('-c', '--config', envvar=VAR_APP_CONFIG_PATH, + required=False, callback=validate_config, + type=click.File('r', encoding=DEFAULT_ENCODING)) +def send(ctx, msg_request: MessageRequest, config: MailerConfig): from .mailer import Mailer mailer = ctx.obj['mailer'] # type: Mailer - mailer.send(rq=msg_request, cfg=None) + mailer.send(rq=msg_request, cfg=config.mail) @cli.command(name='run', help='Run mailer worker processing message jobs.') diff --git a/packages/dsw-mailer/dsw/mailer/config.py b/packages/dsw-mailer/dsw/mailer/config.py index bd8a7e49..5136bd6c 100644 --- a/packages/dsw-mailer/dsw/mailer/config.py +++ b/packages/dsw-mailer/dsw/mailer/config.py @@ -1,19 +1,317 @@ +import enum +import pathlib + from dsw.config import DSWConfigParser -from dsw.config.keys import ConfigKeys +from dsw.config.keys import ConfigKey, ConfigKeys, ConfigKeysContainer, \ + cast_str, cast_bool, cast_optional_str, cast_optional_bool, cast_int, \ + cast_optional_int from dsw.config.model import GeneralConfig, SentryConfig, \ - DatabaseConfig, LoggingConfig, MailConfig + DatabaseConfig, LoggingConfig, ConfigModel, AWSConfig +from dsw.database.model import DBInstanceConfigMail + +from typing import Type + + +class _MailKeys(ConfigKeysContainer): + enabled = ConfigKey( + yaml_path=['mail', 'enabled'], + var_names=['MAIL_ENABLED'], + default=True, + cast=cast_bool, + ) + name = ConfigKey( + yaml_path=['mail', 'name'], + var_names=['MAIL_NAME'], + default='', + cast=cast_str, + ) + email = ConfigKey( + yaml_path=['mail', 'email'], + var_names=['MAIL_EMAIL'], + default='', + cast=cast_str, + ) + provider = ConfigKey( + yaml_path=['mail', 'provider'], + var_names=['MAIL_PROVIDER'], + default='smtp', + cast=cast_str, + ) + rate_limit_window = ConfigKey( + yaml_path=['mail', 'rateLimit', 'window'], + var_names=['MAIL_RATE_LIMIT_WINDOW'], + default=0, + cast=cast_int, + ) + rate_limit_count = ConfigKey( + yaml_path=['mail', 'rateLimit', 'count'], + var_names=['MAIL_RATE_LIMIT_COUNT'], + default=0, + cast=cast_int, + ) + dkim_selector = ConfigKey( + yaml_path=['mail', 'dkim', 'selector'], + var_names=['MAIL_DKIM_SELECTOR'], + default=None, + cast=cast_optional_str, + ) + dkim_privkey_file = ConfigKey( + yaml_path=['mail', 'dkim', 'privkey_file'], + var_names=['MAIL_DKIM_PRIVKEY_FILE'], + default=None, + cast=cast_optional_str, + ) + + +class _MailLegacySMTPKeys(ConfigKeysContainer): + host = ConfigKey( + yaml_path=['mail', 'host'], + var_names=['MAIL_HOST'], + default='', + cast=cast_str, + ) + port = ConfigKey( + yaml_path=['mail', 'port'], + var_names=['MAIL_PORT'], + cast=cast_str, + ) + ssl = ConfigKey( + yaml_path=['mail', 'ssl'], + var_names=[], + cast=cast_optional_str, + ) + security = ConfigKey( + yaml_path=['mail', 'security'], + var_names=['MAIL_SECURITY'], + cast=cast_optional_str, + ) + auth_enabled = ConfigKey( + yaml_path=['mail', 'authEnabled'], + var_names=[], + cast=cast_optional_bool, + ) + username = ConfigKey( + yaml_path=['mail', 'username'], + var_names=['MAIL_USERNAME'], + cast=cast_optional_str, + ) + password = ConfigKey( + yaml_path=['mail', 'password'], + var_names=['MAIL_PASSWORD'], + cast=cast_optional_str, + ) + timeout = ConfigKey( + yaml_path=['mail', 'timeout'], + var_names=['MAIL_TIMEOUT'], + default=5, + cast=cast_int, + ) + + +class _MailSMTPKeys(ConfigKeysContainer): + host = ConfigKey( + yaml_path=['mail', 'smtp', 'host'], + var_names=['MAIL_SMTP_HOST'], + default='', + cast=cast_optional_str, + ) + port = ConfigKey( + yaml_path=['mail', 'smtp', 'port'], + var_names=['MAIL_SMTP_PORT'], + cast=cast_optional_int, + ) + security = ConfigKey( + yaml_path=['mail', 'smtp', 'security'], + var_names=['MAIL_SMTP_SECURITY'], + cast=cast_optional_str, + ) + username = ConfigKey( + yaml_path=['mail', 'smtp', 'username'], + var_names=['MAIL_SMTP_USERNAME'], + cast=cast_optional_str, + ) + password = ConfigKey( + yaml_path=['mail', 'smtp', 'password'], + var_names=['MAIL_SMTP_PASSWORD'], + cast=cast_optional_str, + ) + timeout = ConfigKey( + yaml_path=['mail', 'smtp', 'timeout'], + var_names=['MAIL_TIMEOUT'], + default=5, + cast=cast_int, + ) + + +class _MailAmazonSESKeys(ConfigKeysContainer): + access_key_id = ConfigKey( + yaml_path=['mail', 'amazonSes', 'accessKeyId'], + var_names=['MAIL_SES_ACCESS_KEY_ID'], + cast=cast_optional_str, + ) + secret_access_key = ConfigKey( + yaml_path=['mail', 'amazonSes', 'secretAccessKey'], + var_names=['MAIL_SES_SECRET_ACCESS_KEY'], + cast=cast_optional_str, + ) + region = ConfigKey( + yaml_path=['mail', 'amazonSes', 'region'], + var_names=['MAIL_SES_REGION'], + cast=cast_optional_str, + ) + + +class MailerConfigKeys(ConfigKeys): + mail = _MailKeys + mail_legacy_smtp = _MailLegacySMTPKeys + mail_smtp = _MailSMTPKeys + mail_amazon_ses = _MailAmazonSESKeys + + +class SMTPSecurityMode(enum.Enum): + PLAIN = enum.auto() + SSL = enum.auto() + TLS = enum.auto() + + +class MailProvider(enum.Enum): + NONE = enum.auto() + SMTP = enum.auto() + AMAZON_SES = enum.auto() + + +class MailSMTPConfig: + + def __init__(self, host: str | None = None, port: int | None = None, + security: str | None = None, ssl: bool | None = None, + username: str | None = None, password: str | None = None, + auth_enabled: bool | None = None, timeout: int = 10): + self.host = host + self.security = SMTPSecurityMode.PLAIN # type: SMTPSecurityMode + if security is not None and security.upper() in SMTPSecurityMode: + self.security = SMTPSecurityMode[security.upper()] + elif ssl is not None: + self.security = SMTPSecurityMode.SSL if ssl else SMTPSecurityMode.PLAIN + self.port = port or self._default_port() + self.auth = auth_enabled + if self.auth is None: + self.auth = username is not None and password is not None + self.username = username + self.password = password + self.timeout = timeout + + @property + def login_user(self) -> str: + return self.username or '' + + @property + def login_password(self) -> str: + return self.password or '' + + @property + def is_plain(self): + return self.security == SMTPSecurityMode.PLAIN + + @property + def is_ssl(self): + return self.security == SMTPSecurityMode.SSL + + @property + def is_tls(self): + return self.security == SMTPSecurityMode.TLS + + def _default_port(self) -> int: + if self.is_plain: + return 25 + if self.is_ssl: + return 465 + return 587 + + def has_credentials(self) -> bool: + return self.username is not None and self.password is not None + + +class MailAmazonSESConfig: + + def __init__(self, access_key_id: str | None = None, + secret_access_key: str | None = None, + region: str | None = None): + self.access_key_id = access_key_id + self.secret_access_key = secret_access_key + self.region = region + + def has_credentials(self) -> bool: + return self.access_key_id is not None and self.secret_access_key is not None + + +class MailConfig(ConfigModel): + + def __init__(self, enabled: bool, name: str, email: str, + provider: str, smtp: MailSMTPConfig, amazon_ses: MailAmazonSESConfig, + rate_limit_window: int, rate_limit_count: int, + dkim_selector: str | None = None, dkim_privkey_file: str | None = None): + self.enabled = enabled + self.name = name + self.email = email + + if provider.lower() == 'smtp': + self.provider = MailProvider.SMTP + elif provider.lower() in ['ses', 'amazon_ses', 'amazonses']: + self.provider = MailProvider.AMAZON_SES + else: + raise ValueError(f'Unknown mail provider: {provider}') + self.smtp = smtp + self.amazon_ses = amazon_ses + + self.rate_limit_window = rate_limit_window + self.rate_limit_count = rate_limit_count + self.dkim_selector = dkim_selector + self.dkim_privkey_file = dkim_privkey_file + self.dkim_privkey = b'' + + def load_dkim_privkey(self): + if self.dkim_privkey_file is not None: + self.dkim_privkey = pathlib.Path(self.dkim_privkey_file).read_bytes() + self.dkim_privkey = self.dkim_privkey.replace(b'\r\n', b'\n') + + def update_aws(self, aws: AWSConfig): + if self.provider == MailProvider.AMAZON_SES: + if not self.amazon_ses.has_credentials(): + self.amazon_ses.access_key_id = aws.access_key_id + self.amazon_ses.secret_access_key = aws.secret_access_key + if self.amazon_ses.region is None: + self.amazon_ses.region = aws.region + + @property + def use_dkim(self): + return self.dkim_selector is not None and len(self.dkim_privkey) > 0 + + def __str__(self): + return f'MailConfig\n' \ + f'- enabled = {self.enabled}\n' \ + f'- name = {self.name}\n' \ + f'- email = {self.email}\n' \ + f'- provider = {self.provider}\n' \ + f'- rate_limit_window = {self.rate_limit_window}\n' \ + f'- rate_limit_count = {self.rate_limit_count}\n' \ + f'- dkim_selector = {self.dkim_selector}\n' \ + f'- dkim_privkey_file = {self.dkim_privkey_file}\n' class MailerConfig: def __init__(self, db: DatabaseConfig, log: LoggingConfig, mail: MailConfig, sentry: SentryConfig, - general: GeneralConfig): + general: GeneralConfig, aws: AWSConfig): self.db = db self.log = log self.mail = mail self.sentry = sentry self.general = general + self.aws = aws + + # Use AWS credentials for Amazon SES if not provided + self.mail.update_aws(aws) def __str__(self): return f'MailerConfig\n' \ @@ -29,10 +327,49 @@ def __str__(self): class MailerConfigParser(DSWConfigParser): def __init__(self): - ConfigKeys.mail.email.required = True - ConfigKeys.mail.host.required = True - ConfigKeys.mail.port.required = True - super().__init__(keys=ConfigKeys) + super().__init__(keys=MailerConfigKeys) + self.keys = MailerConfigKeys # type: Type[MailerConfigKeys] + + @property + def mail(self): + smtp = MailSMTPConfig( + host=self.get(self.keys.mail_smtp.host), + port=self.get(self.keys.mail_smtp.port), + security=self.get(self.keys.mail_smtp.security), + username=self.get(self.keys.mail_smtp.username), + password=self.get(self.keys.mail_smtp.password), + timeout=int(self.get(self.keys.mail_smtp.timeout)), + ) + if smtp.host == '': + smtp = MailSMTPConfig( + host=self.get(self.keys.mail_legacy_smtp.host), + port=self.get(self.keys.mail_legacy_smtp.port), + security=self.get(self.keys.mail_legacy_smtp.security), + auth_enabled=self.get(self.keys.mail_legacy_smtp.auth_enabled), + username=self.get(self.keys.mail_legacy_smtp.username), + password=self.get(self.keys.mail_legacy_smtp.password), + ssl=self.get(self.keys.mail_legacy_smtp.ssl), + timeout=int(self.get(self.keys.mail_legacy_smtp.timeout)), + ) + + amazon_ses = MailAmazonSESConfig( + access_key_id=self.get(self.keys.mail_amazon_ses.access_key_id), + secret_access_key=self.get(self.keys.mail_amazon_ses.secret_access_key), + region=self.get(self.keys.mail_amazon_ses.region), + ) + + return MailConfig( + enabled=self.get(self.keys.mail.enabled), + name=self.get(self.keys.mail.name), + email=self.get(self.keys.mail.email), + provider=self.get(self.keys.mail.provider), + smtp=smtp, + amazon_ses=amazon_ses, + rate_limit_window=int(self.get(self.keys.mail.rate_limit_window)), + rate_limit_count=int(self.get(self.keys.mail.rate_limit_count)), + dkim_selector=self.get(self.keys.mail.dkim_selector), + dkim_privkey_file=self.get(self.keys.mail.dkim_privkey_file), + ) @property def config(self) -> MailerConfig: @@ -42,6 +379,54 @@ def config(self) -> MailerConfig: mail=self.mail, sentry=self.sentry, general=self.general, + aws=self.aws, ) cfg.mail.load_dkim_privkey() return cfg + + +def merge_mail_configs(cfg: MailerConfig, db_cfg: DBInstanceConfigMail | None) -> MailConfig: + if db_cfg is None: + return cfg.mail + + smtp = MailSMTPConfig() + amazon_ses = MailAmazonSESConfig() + if db_cfg.provider.lower() == 'smtp': + if db_cfg.smtp_host is None: + smtp.host = cfg.mail.smtp.host + smtp.port = cfg.mail.smtp.port + smtp.security = cfg.mail.smtp.security + smtp.username = cfg.mail.smtp.username + smtp.password = cfg.mail.smtp.password + smtp.timeout = cfg.mail.smtp.timeout + else: + smtp.host = db_cfg.smtp_host + smtp.port = db_cfg.smtp_port + smtp.security = db_cfg.smtp_security + smtp.username = db_cfg.smtp_username + smtp.password = db_cfg.smtp_password + smtp.timeout = db_cfg.timeout + elif db_cfg.provider.lower() == 'amazonses': + if db_cfg.aws_access_key_id is None and db_cfg.aws_secret_access_key is None: + amazon_ses.access_key_id = cfg.mail.amazon_ses.access_key_id + amazon_ses.secret_access_key = cfg.mail.amazon_ses.secret_access_key + amazon_ses.region = cfg.mail.amazon_ses.region + else: + amazon_ses.access_key_id = db_cfg.aws_access_key_id + amazon_ses.secret_access_key = db_cfg.aws_secret_access_key + amazon_ses.region = db_cfg.aws_region + + result = MailConfig( + enabled=cfg.mail.enabled, + name=db_cfg.sender_name, + email=db_cfg.sender_email, + provider=db_cfg.provider, + smtp=smtp, + amazon_ses=amazon_ses, + rate_limit_window=db_cfg.rate_limit_window, + rate_limit_count=db_cfg.rate_limit_count, + dkim_privkey_file=None, + dkim_selector=None, + ) + result.update_aws(cfg.aws) + return result diff --git a/packages/dsw-mailer/dsw/mailer/consts.py b/packages/dsw-mailer/dsw/mailer/consts.py index 8c29f594..aa966902 100644 --- a/packages/dsw-mailer/dsw/mailer/consts.py +++ b/packages/dsw-mailer/dsw/mailer/consts.py @@ -5,7 +5,7 @@ DEFAULT_ENCODING = 'utf-8' NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'dsw-mailer' -VERSION = '4.8.1' +VERSION = '4.9.0' VAR_APP_CONFIG_PATH = 'APPLICATION_CONFIG_PATH' VAR_WORKDIR_PATH = 'WORKDIR_PATH' diff --git a/packages/dsw-mailer/dsw/mailer/context.py b/packages/dsw-mailer/dsw/mailer/context.py index 2c3fcad4..e7c6aac0 100644 --- a/packages/dsw-mailer/dsw/mailer/context.py +++ b/packages/dsw-mailer/dsw/mailer/context.py @@ -1,12 +1,11 @@ import pathlib -from typing import Optional, TYPE_CHECKING +from typing import TYPE_CHECKING from .templates import TemplateRegistry if TYPE_CHECKING: from .config import MailerConfig - from .connection import SMTPSender from dsw.database import Database @@ -18,10 +17,9 @@ def __init__(self): class AppContext: - def __init__(self, db, cfg, sender, workdir): + def __init__(self, db, cfg, workdir): self.db = db # type: Database self.cfg = cfg # type: MailerConfig - self.sender = sender # type: SMTPSender self.workdir = workdir # type: pathlib.Path @@ -49,7 +47,7 @@ def reset_ids(self): class Context: - _instance = None # type: Optional[_Context] + _instance = None # type: _Context | None @classmethod def get(cls) -> _Context: @@ -58,13 +56,12 @@ def get(cls) -> _Context: return cls._instance @classmethod - def initialize(cls, db, config, sender, workdir): + def initialize(cls, db, config, workdir): cls._instance = _Context( app=AppContext( db=db, cfg=config, workdir=workdir, - sender=sender, ), job=JobContext( trace_id='-', diff --git a/packages/dsw-mailer/dsw/mailer/mailer.py b/packages/dsw-mailer/dsw/mailer/mailer.py index 1bedd99b..db43ae30 100644 --- a/packages/dsw-mailer/dsw/mailer/mailer.py +++ b/packages/dsw-mailer/dsw/mailer/mailer.py @@ -6,17 +6,15 @@ import time import urllib.parse -from typing import Optional - from dsw.command_queue import CommandWorker, CommandQueue from dsw.config.sentry import SentryReporter from dsw.database.database import Database -from dsw.database.model import PersistentCommand, DBInstanceConfigMail +from dsw.database.model import PersistentCommand from .build_info import BUILD_INFO -from .config import MailerConfig, MailConfig +from .config import MailerConfig, MailConfig, merge_mail_configs from .consts import PROG_NAME -from .smtp import SMTPSender +from .sender import send from .consts import COMPONENT_NAME, CMD_CHANNEL, CMD_COMPONENT, \ CMD_FUNCTION from .context import Context @@ -44,7 +42,6 @@ def _init_context(self, workdir: pathlib.Path): config=self.cfg, workdir=workdir, db=Database(cfg=self.cfg.db, connect=False), - sender=SMTPSender(cfg=self.cfg.mail), ) SentryReporter.initialize( dsn=self.cfg.sentry.workers_dsn, @@ -109,8 +106,9 @@ def work(self, cmd: PersistentCommand): if tenant_cfg is not None: rq.style.from_dict(tenant_cfg.look_and_feel) # get mailer config from DB - mail_cfg = _transform_mail_config( - cfg=app_ctx.db.get_mail_config(tenant_uuid=cmd.tenant_uuid), + mail_cfg = merge_mail_configs( + cfg=self.cfg, + db_cfg=app_ctx.db.get_mail_config(tenant_uuid=cmd.tenant_uuid), ) LOG.debug(f'Mail config from DB: {mail_cfg}') # client URL @@ -127,7 +125,7 @@ def process_exception(self, e: Exception): LOG.info('Failed with unexpected error', exc_info=e) SentryReporter.capture_exception(e) - def send(self, rq: MessageRequest, cfg: Optional[MailConfig]): + def send(self, rq: MessageRequest, cfg: MailConfig): LOG.info(f'Sending request: {rq.template_name} ({rq.id})') # get template if not self.ctx.templates.has_template_for(rq): @@ -137,32 +135,10 @@ def send(self, rq: MessageRequest, cfg: Optional[MailConfig]): msg = self.ctx.templates.render(rq, cfg) # send LOG.info(f'Sending message: {rq.template_name}') - self.ctx.app.sender.send(msg, cfg) + send(msg, cfg) LOG.info('Message sent successfully') -def _transform_mail_config(cfg: Optional[DBInstanceConfigMail]) -> Optional[MailConfig]: - if cfg is None: - return None - return MailConfig( - enabled=cfg.enabled, - name=cfg.sender_name, - email=cfg.sender_email, - host=cfg.host, - port=cfg.port, - security=cfg.security, - username=cfg.username, - password=cfg.password, - rate_limit_window=cfg.rate_limit_window, - rate_limit_count=cfg.rate_limit_count, - timeout=cfg.timeout, - ssl=None, - auth_enabled=None, - dkim_privkey_file=None, - dkim_selector=None, - ) - - class RateLimiter: def __init__(self, window: int, count: int): diff --git a/packages/dsw-mailer/dsw/mailer/model.py b/packages/dsw-mailer/dsw/mailer/model.py index bb6fd704..722f419e 100644 --- a/packages/dsw-mailer/dsw/mailer/model.py +++ b/packages/dsw-mailer/dsw/mailer/model.py @@ -1,8 +1,6 @@ import os import re -from typing import Optional - class Color: DEFAULT_PRIMARY_HEX = os.getenv('DEFAULT_PRIMARY_COLOR', '#0033aa') @@ -25,7 +23,7 @@ def __init__(self, color_hex: str = '#000000', default: str = '#000000'): self.red, self.green, self.blue = tuple(int(h[i:i+2], 16) for i in (0, 2, 4)) @staticmethod - def parse_color_to_hex(color: str) -> Optional[str]: + def parse_color_to_hex(color: str) -> str | None: color = color.strip() if re.match(r'^#[0-9a-fA-F]{6}$', color): return color @@ -78,13 +76,13 @@ def __str__(self): class StyleConfig: _DEFAULT = None - def __init__(self, logo_url: Optional[str], primary_color: str, + def __init__(self, logo_url: str | None, primary_color: str, illustrations_color: str): self.logo_url = logo_url self.primary_color = Color(primary_color, Color.DEFAULT_PRIMARY_HEX) self.illustrations_color = Color(illustrations_color, Color.DEFAULT_ILLUSTRATIONS_HEX) - def from_dict(self, data: Optional[dict]): + def from_dict(self, data: dict | None): data = data or dict() if data.get('logoUrl', None) is not None: self.logo_url = data.get('logoUrl') @@ -159,9 +157,9 @@ def load_from_file(data: dict) -> 'TemplateDescriptorPart': class TemplateDescriptor: def __init__(self, message_id: str, subject: str, subject_prefix: bool, - default_sender_name: Optional[str], language: str, - importance: str, sensitivity: Optional[str], - priority: Optional[str]): + default_sender_name: str | None, language: str, + importance: str, sensitivity: str | None, + priority: str | None): self.id = message_id self.subject = subject self.use_subject_prefix = subject_prefix @@ -194,13 +192,13 @@ def load_from_file(data: dict) -> 'TemplateDescriptor': class MessageRequest: def __init__(self, message_id: str, template_name: str, trigger: str, - ctx: dict, recipients: list[str], style: Optional[StyleConfig] = None): + ctx: dict, recipients: list[str], style: StyleConfig | None = None): self.id = message_id self.template_name = template_name self.trigger = trigger self.ctx = ctx self.recipients = recipients - self.domain = None # type: Optional[str] + self.domain = None # type: str | None self.client_url = '' # type: str self.style = style or StyleConfig.default() self.ctx['style'] = self.style @@ -225,19 +223,19 @@ class MailMessage: def __init__(self): self.from_mail = '' # type: str - self.from_name = None # type: Optional[str] + self.from_name = None # type: str | None self.recipients = list() # type: list[str] self.subject = '' # type: str - self.plain_body = None # type: Optional[str] - self.html_body = None # type: Optional[str] + self.plain_body = None # type: str | None + self.html_body = None # type: str | None self.html_images = list() # type: list[MailAttachment] self.attachments = list() # type: list[MailAttachment] - self.msg_id = None # type: Optional[str] - self.msg_domain = None # type: Optional[str] + self.msg_id = None # type: str | None + self.msg_domain = None # type: str | None self.language = 'en' # type: str self.importance = 'normal' # type: str - self.sensitivity = None # type: Optional[str] - self.priority = None # type: Optional[str] + self.sensitivity = None # type: str | None + self.priority = None # type: str | None self.client_url = '' # type: str diff --git a/packages/dsw-mailer/dsw/mailer/sender/__init__.py b/packages/dsw-mailer/dsw/mailer/sender/__init__.py new file mode 100644 index 00000000..a94b377a --- /dev/null +++ b/packages/dsw-mailer/dsw/mailer/sender/__init__.py @@ -0,0 +1,3 @@ +from .dispatch import send + +__all__ = ['send'] diff --git a/packages/dsw-mailer/dsw/mailer/sender/amazon_ses.py b/packages/dsw-mailer/dsw/mailer/sender/amazon_ses.py new file mode 100644 index 00000000..647ac2ae --- /dev/null +++ b/packages/dsw-mailer/dsw/mailer/sender/amazon_ses.py @@ -0,0 +1,39 @@ +import boto3 +import logging + +from .base import BaseMailSender +from ..config import MailConfig +from ..model import MailMessage + + +LOG = logging.getLogger(__name__) + + +class AmazonSESSender(BaseMailSender): + + @staticmethod + def validate_config(cfg: MailConfig): + if not cfg.amazon_ses.has_credentials(): + raise ValueError('Missing credentials for Amazon SES') + if not cfg.amazon_ses.region: + raise ValueError('Missing region for Amazon SES') + + def send(self, message: MailMessage): + LOG.info(f'Sending via Amazon SES (region {self.cfg.amazon_ses.region})') + self._send(message, self.cfg) + + def _send(self, mail: MailMessage, cfg: MailConfig): + ses = boto3.client( + 'ses', + region_name=cfg.amazon_ses.region, + aws_access_key_id=cfg.amazon_ses.access_key_id, + aws_secret_access_key=cfg.amazon_ses.secret_access_key, + ) + msg = self._convert_email(mail) + return ses.send_raw_email( + Source=mail.from_mail, + Destinations=mail.recipients, + RawMessage={ + 'Data': msg.as_string().encode('us-ascii'), + }, + ) diff --git a/packages/dsw-mailer/dsw/mailer/smtp.py b/packages/dsw-mailer/dsw/mailer/sender/base.py similarity index 58% rename from packages/dsw-mailer/dsw/mailer/smtp.py rename to packages/dsw-mailer/dsw/mailer/sender/base.py index a04e7f31..4ff86200 100644 --- a/packages/dsw-mailer/dsw/mailer/smtp.py +++ b/packages/dsw-mailer/dsw/mailer/sender/base.py @@ -1,95 +1,89 @@ +import abc import datetime import dkim import logging import pathvalidate -import smtplib -import ssl -import tenacity from email import encoders from email.mime.base import MIMEBase from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formataddr, format_datetime, make_msgid -from typing import Optional -from dsw.config.model import MailConfig +from ..config import MailConfig +from ..consts import DEFAULT_ENCODING +from ..model import MailMessage, MailAttachment -from .consts import DEFAULT_ENCODING -from .model import MailMessage, MailAttachment - -RETRY_SMTP_MULTIPLIER = 0.5 -RETRY_SMTP_TRIES = 3 LOG = logging.getLogger(__name__) -class SMTPSender: +class BaseMailSender(abc.ABC): - def __init__(self, cfg: MailConfig): - self.default_cfg = cfg + def __init__(self): + self._cfg = None # type: MailConfig | None - @tenacity.retry( - reraise=True, - wait=tenacity.wait_exponential(multiplier=RETRY_SMTP_MULTIPLIER), - stop=tenacity.stop_after_attempt(RETRY_SMTP_TRIES), - before=tenacity.before_log(LOG, logging.DEBUG), - after=tenacity.after_log(LOG, logging.DEBUG), - ) - def send(self, message: MailMessage, cfg: Optional[MailConfig]): - used_cfg = cfg or self.default_cfg - if not used_cfg.enabled: - LOG.info('Not actually sending email (enabled=False)') - return - LOG.info(f'Sending via SMTP: {used_cfg.host}:{used_cfg.port}') - self._send(message, used_cfg) + @property + def cfg(self) -> MailConfig: + if self._cfg is None: + raise RuntimeError('Mail sender not prepared') + return self._cfg - @classmethod - def _send(cls, mail: MailMessage, cfg: MailConfig): - if cfg.is_ssl: - return cls._send_smtp_ssl(mail=mail, cfg=cfg) - return cls._send_smtp(mail=mail, cfg=cfg) + def prepare(self, cfg: MailConfig): + self.validate_config(cfg) + self._cfg = cfg - @classmethod - def _send_smtp_ssl(cls, mail: MailMessage, cfg: MailConfig): - context = ssl.create_default_context() - with smtplib.SMTP_SSL( - host=cfg.host, - port=cfg.port, - context=context, - timeout=cfg.timeout, - ) as server: - if cfg.auth: - server.login( - user=cfg.login_user, - password=cfg.login_password, - ) - return server.send_message( - msg=cls._convert_email(mail, cfg), - from_addr=formataddr((mail.from_name, mail.from_mail)), - to_addrs=mail.recipients, - ) + @staticmethod + @abc.abstractmethod + def validate_config(cfg: MailConfig): + pass - @classmethod - def _send_smtp(cls, mail: MailMessage, cfg: MailConfig): - context = ssl.create_default_context() - with smtplib.SMTP( - host=cfg.host, - port=cfg.port, - timeout=cfg.timeout, - ) as server: - if cfg.is_tls: - server.starttls(context=context) - if cfg.auth: - server.login( - user=cfg.login_user, - password=cfg.login_password, - ) - return server.send_message( - msg=cls._convert_email(mail, cfg), - from_addr=formataddr((mail.from_name, mail.from_mail)), - to_addrs=mail.recipients, - ) + @abc.abstractmethod + def send(self, message: MailMessage): + ... + + def _convert_email(self, mail: MailMessage) -> MIMEBase: + msg = self._convert_txt_parts(mail) + if len(mail.attachments) > 0: + txt = msg + msg = MIMEMultipart('mixed') + msg.attach(txt) + for attachment in mail.attachments: + msg.attach(self._convert_attachment(attachment)) + + headers = [] # type: list[bytes] + + def add_header(name: str, value: str): + msg.add_header(name, value) + headers.append(name.encode(encoding=DEFAULT_ENCODING)) + + add_header('From', formataddr((mail.from_name, mail.from_mail))) + add_header('To', ', '.join(mail.recipients)) + add_header('Subject', mail.subject) + add_header('Date', format_datetime(dt=datetime.datetime.now(tz=datetime.UTC))) + add_header('Message-ID', make_msgid(idstring=mail.msg_id, domain=mail.msg_domain)) + add_header('Language', mail.language) + add_header('Importance', mail.importance) + add_header('List-Unsubscribe', f'{mail.client_url}/users/edit/current') + if mail.sensitivity is not None: + add_header('Sensitivity', mail.sensitivity) + if mail.priority is not None: + add_header('Priority', mail.priority) + + if self.cfg.dkim_selector and self.cfg.dkim_privkey: + sender_domain = mail.from_mail.split('@')[-1] + signature = dkim.sign( + message=msg.as_bytes(), + selector=self.cfg.dkim_selector.encode(), + domain=sender_domain.encode(), + privkey=self.cfg.dkim_privkey, + include_headers=headers, + ).decode() + if signature.startswith('DKIM-Signature: '): + signature = signature[len('DKIM-Signature: '):] + msg.add_header('DKIM-Signature', signature) + + return msg @staticmethod def _convert_inline_image(image: MailAttachment) -> MIMEBase: @@ -144,46 +138,13 @@ def _convert_attachment(attachment: MailAttachment) -> MIMEBase: part.add_header('Content-Disposition', f'attachment; filename={filename}') return part - @classmethod - def _convert_email(cls, mail: MailMessage, cfg: MailConfig) -> MIMEBase: - msg = cls._convert_txt_parts(mail) - if len(mail.attachments) > 0: - txt = msg - msg = MIMEMultipart('mixed') - msg.attach(txt) - for attachment in mail.attachments: - msg.attach(cls._convert_attachment(attachment)) - headers = [] # type: list[bytes] - - def add_header(name: str, value: str): - msg.add_header(name, value) - headers.append(name.encode(encoding=DEFAULT_ENCODING)) - - add_header('From', formataddr((mail.from_name, mail.from_mail))) - add_header('To', ', '.join(mail.recipients)) - add_header('Subject', mail.subject) - add_header('Date', format_datetime(dt=datetime.datetime.now(tz=datetime.UTC))) - add_header('Message-ID', make_msgid(idstring=mail.msg_id, domain=mail.msg_domain)) - add_header('Language', mail.language) - add_header('Importance', mail.importance) - add_header('List-Unsubscribe', f'{mail.client_url}/users/edit/current') - if mail.sensitivity is not None: - add_header('Sensitivity', mail.sensitivity) - if mail.priority is not None: - add_header('Priority', mail.priority) +class NoProviderSender(BaseMailSender): - if cfg.dkim_selector and cfg.dkim_privkey: - sender_domain = mail.from_mail.split('@')[-1] - signature = dkim.sign( - message=msg.as_bytes(), - selector=cfg.dkim_selector.encode(), - domain=sender_domain.encode(), - privkey=cfg.dkim_privkey, - include_headers=headers, - ).decode() - if signature.startswith('DKIM-Signature: '): - signature = signature[len('DKIM-Signature: '):] - msg.add_header('DKIM-Signature', signature) + @staticmethod + def validate_config(cfg: MailConfig): + pass - return msg + def send(self, message: MailMessage): + LOG.info('No provider configured, not sending anything') + return diff --git a/packages/dsw-mailer/dsw/mailer/sender/dispatch.py b/packages/dsw-mailer/dsw/mailer/sender/dispatch.py new file mode 100644 index 00000000..37f5ce53 --- /dev/null +++ b/packages/dsw-mailer/dsw/mailer/sender/dispatch.py @@ -0,0 +1,37 @@ +import logging + +from .base import BaseMailSender, NoProviderSender +from .amazon_ses import AmazonSESSender +from .smtp import SMTPSender + +from ..config import MailConfig, MailProvider +from ..model import MailMessage + + +LOG = logging.getLogger(__name__) + + +SENDERS = { + MailProvider.SMTP: SMTPSender(), + MailProvider.AMAZON_SES: AmazonSESSender(), + MailProvider.NONE: NoProviderSender(), +} # type: dict[MailProvider, BaseMailSender] + + +def get_sender(cfg: MailConfig) -> BaseMailSender: + if cfg.provider not in SENDERS: + raise ValueError(f'Unsupported mail provider ' + f'(no sender available): {cfg.provider}') + return SENDERS[cfg.provider] + + +def send(message: MailMessage, cfg: MailConfig): + if cfg.enabled is False: + LOG.info('Mail sending is disabled, skipping...') + return + sender = get_sender(cfg) + sender.prepare(cfg) + sender.send(message) + + +__all__ = ['get_sender', 'send', 'SENDERS', 'BaseMailSender'] diff --git a/packages/dsw-mailer/dsw/mailer/sender/smtp.py b/packages/dsw-mailer/dsw/mailer/sender/smtp.py new file mode 100644 index 00000000..0dc6a477 --- /dev/null +++ b/packages/dsw-mailer/dsw/mailer/sender/smtp.py @@ -0,0 +1,78 @@ +import logging +import smtplib +import ssl +import tenacity + +from email.utils import formataddr + +from .base import BaseMailSender +from ..config import MailConfig +from ..model import MailMessage + + +RETRY_SMTP_MULTIPLIER = 0.5 +RETRY_SMTP_TRIES = 3 +LOG = logging.getLogger(__name__) + + +class SMTPSender(BaseMailSender): + + @staticmethod + def validate_config(cfg: MailConfig): + if not cfg.smtp.host: + raise ValueError('Missing host for SMTP') + if not cfg.smtp.port: + raise ValueError('Missing port for SMTP') + + @tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=RETRY_SMTP_MULTIPLIER), + stop=tenacity.stop_after_attempt(RETRY_SMTP_TRIES), + before=tenacity.before_log(LOG, logging.DEBUG), + after=tenacity.after_log(LOG, logging.DEBUG), + ) + def send(self, message: MailMessage): + LOG.info(f'Sending via SMTP (server {self.cfg.smtp.host}:{self.cfg.smtp.port})') + if self.cfg.smtp.is_ssl: + self._send_smtp_ssl(mail=message) + else: + self._send_smtp(mail=message) + + def _send_smtp_ssl(self, mail: MailMessage): + context = ssl.create_default_context() + with smtplib.SMTP_SSL( + host=self.cfg.smtp.host or 'localhost', + port=self.cfg.smtp.port, + context=context, + timeout=self.cfg.smtp.timeout, + ) as server: + if self.cfg.smtp.auth: + server.login( + user=self.cfg.smtp.login_user, + password=self.cfg.smtp.login_password, + ) + return server.send_message( + msg=self._convert_email(mail), + from_addr=formataddr((mail.from_name, mail.from_mail)), + to_addrs=mail.recipients, + ) + + def _send_smtp(self, mail: MailMessage): + context = ssl.create_default_context() + with smtplib.SMTP( + host=self.cfg.smtp.host or 'localhost', + port=self.cfg.smtp.port, + timeout=self.cfg.smtp.timeout, + ) as server: + if self.cfg.smtp.is_tls: + server.starttls(context=context) + if self.cfg.smtp.auth: + server.login( + user=self.cfg.smtp.login_user, + password=self.cfg.smtp.login_password, + ) + return server.send_message( + msg=self._convert_email(mail), + from_addr=formataddr((mail.from_name, mail.from_mail)), + to_addrs=mail.recipients, + ) diff --git a/packages/dsw-mailer/dsw/mailer/templates.py b/packages/dsw-mailer/dsw/mailer/templates.py index 4efef4df..8f1858f2 100644 --- a/packages/dsw-mailer/dsw/mailer/templates.py +++ b/packages/dsw-mailer/dsw/mailer/templates.py @@ -4,9 +4,10 @@ import jinja2.sandbox import json import logging +import markdown +import markupsafe import pathlib - -from typing import Optional, Union +import re from .config import MailerConfig, MailConfig from .consts import DEFAULT_ENCODING @@ -20,8 +21,8 @@ class MailTemplate: def __init__(self, name: str, descriptor: TemplateDescriptor, - html_template: Optional[jinja2.Template], - plain_template: Optional[jinja2.Template]): + html_template: jinja2.Template | None, + plain_template: jinja2.Template | None): self.name = name self.descriptor = descriptor self.html_template = html_template @@ -29,7 +30,7 @@ def __init__(self, name: str, descriptor: TemplateDescriptor, self.attachments = list() # type: list[MailAttachment] self.html_images = list() # type: list[MailAttachment] - def render(self, rq: MessageRequest, mail_name: Optional[str], mail_from: str) -> MailMessage: + def render(self, rq: MessageRequest, mail_name: str | None, mail_from: str) -> MailMessage: ctx = rq.ctx msg = MailMessage() msg.recipients = rq.recipients @@ -77,9 +78,11 @@ def __init__(self, cfg: MailerConfig, workdir: pathlib.Path): def _set_filters(self): self.j2_env.filters.update({ 'datetime_format': datetime_format, + 'markdown': xmarkdown, + 'no_markdown': remove_markdown, }) - def _load_jinja2(self, file_path: pathlib.Path) -> Optional[jinja2.Template]: + def _load_jinja2(self, file_path: pathlib.Path) -> jinja2.Template | None: if file_path.exists() and file_path.is_file(): return self.j2_env.get_template( name=str(file_path.relative_to(self.workdir).as_posix()), @@ -88,7 +91,7 @@ def _load_jinja2(self, file_path: pathlib.Path) -> Optional[jinja2.Template]: @staticmethod def _load_attachment(template_path: pathlib.Path, - part: TemplateDescriptorPart) -> Optional[MailAttachment]: + part: TemplateDescriptorPart) -> MailAttachment | None: file_path = template_path / part.file if file_path.exists() and file_path.is_file(): binary_data = file_path.read_bytes() @@ -100,7 +103,7 @@ def _load_attachment(template_path: pathlib.Path, return None @staticmethod - def _load_descriptor(path: pathlib.Path) -> Optional[TemplateDescriptor]: + def _load_descriptor(path: pathlib.Path) -> TemplateDescriptor | None: if not path.exists() or not path.is_file(): return None try: @@ -112,7 +115,7 @@ def _load_descriptor(path: pathlib.Path) -> Optional[TemplateDescriptor]: return None def _load_template(self, path: pathlib.Path, - descriptor: TemplateDescriptor) -> Optional[MailTemplate]: + descriptor: TemplateDescriptor) -> MailTemplate | None: html_template = None plain_template = None attachments = list() @@ -164,9 +167,67 @@ def render(self, rq: MessageRequest, cfg: MailConfig) -> MailMessage: ) -def datetime_format(iso_timestamp: Union[None, datetime.datetime, str], fmt: str): +def datetime_format(iso_timestamp: None | datetime.datetime | str, fmt: str): if iso_timestamp is None: return '' if not isinstance(iso_timestamp, datetime.datetime): iso_timestamp = dateutil.parser.isoparse(iso_timestamp) return iso_timestamp.strftime(fmt) + + +class DSWMarkdownExt(markdown.extensions.Extension): + def extendMarkdown(self, md): + md.preprocessors.register(DSWMarkdownProcessor(md), 'dsw_markdown', 27) + md.registerExtension(self) + + +class DSWMarkdownProcessor(markdown.preprocessors.Preprocessor): + + def __init__(self, md): + super().__init__(md) + self.LI_RE = re.compile(r'^[ ]*((\d+\.)|[*+-])[ ]+.*') + + def run(self, lines): + prev_li = False + new_lines = [] + + for line in lines: + # Add line break before the first list item + if self.LI_RE.match(line): + if not prev_li: + new_lines.append('') + prev_li = True + elif line == '': + prev_li = False + + # Replace trailing un-escaped backslash with (supported) two spaces + _line = line.rstrip('\\') + if line[-1:] == '\\' and (len(line) - len(_line)) % 2 == 1: + new_lines.append(f'{line[:-1]} ') + continue + + new_lines.append(line) + + return new_lines + + +def xmarkdown(md_text: str): + if md_text is None: + return '' + return markupsafe.Markup(markdown.markdown( + text=md_text, + extensions=[ + DSWMarkdownExt(), + ] + )) + + +def remove_markdown(md_text: str): + if md_text is None: + return '' + return re.sub(r'<[^>]*>', '', markdown.markdown( + text=md_text, + extensions=[ + DSWMarkdownExt(), + ] + )) diff --git a/packages/dsw-mailer/pyproject.toml b/packages/dsw-mailer/pyproject.toml index bbc824f1..cdcd03eb 100644 --- a/packages/dsw-mailer/pyproject.toml +++ b/packages/dsw-mailer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-mailer' -version = "4.8.1" +version = "4.9.0" description = 'Worker for sending email notifications' readme = 'README.md' keywords = ['email', 'jinja2', 'notification', 'template'] @@ -23,17 +23,19 @@ classifiers = [ ] requires-python = '>=3.10, <4' dependencies = [ + 'boto3', 'click', 'dkimpy', 'Jinja2', + 'Markdown', 'pathvalidate', 'python-dateutil', 'sentry-sdk', 'tenacity', # DSW - "dsw-command-queue==4.8.1", - "dsw-config==4.8.1", - "dsw-database==4.8.1", + "dsw-command-queue==4.9.0", + "dsw-config==4.9.0", + "dsw-database==4.9.0", ] [project.urls] diff --git a/packages/dsw-mailer/requirements.txt b/packages/dsw-mailer/requirements.txt index 7e21e877..6948d500 100644 --- a/packages/dsw-mailer/requirements.txt +++ b/packages/dsw-mailer/requirements.txt @@ -1,17 +1,22 @@ -certifi==2024.6.2 +boto3==1.34.139 +botocore==1.34.139 +certifi==2024.7.4 click==8.1.7 -dkimpy==1.1.7 +dkimpy==1.1.8 dnspython==2.6.1 Jinja2==3.1.4 +Markdown==3.6 +jmespath==1.0.1 MarkupSafe==2.1.5 pathvalidate==3.2.0 -psycopg==3.1.19 -psycopg-binary==3.1.19 +psycopg==3.2.1 +psycopg-binary==3.2.1 python-dateutil==2.9.0 PyYAML==6.0.1 -sentry-sdk==2.6.0 +s3transfer==0.10.2 +sentry-sdk==2.8.0 six==1.16.0 -tenacity==8.4.1 +tenacity==8.5.0 typing_extensions==4.12.2 tzdata==2024.1 urllib3==2.2.2 diff --git a/packages/dsw-mailer/templates/_common/style.css b/packages/dsw-mailer/templates/_common/style.css index 75248123..aada15d8 100644 --- a/packages/dsw-mailer/templates/_common/style.css +++ b/packages/dsw-mailer/templates/_common/style.css @@ -199,7 +199,7 @@ h4 { .headerContainer .dswTextContent, .headerContainer .dswTextContent p { - color: #666666; + color: #212529; font-family: Helvetica; font-size: 16px; line-height: 150%; @@ -239,7 +239,7 @@ h4 { .bodyContainer .dswTextContent, .bodyContainer .dswTextContent p { - color: #666666; + color: #212529; font-family: Helvetica; font-size: 16px; line-height: 150%; diff --git a/packages/dsw-mailer/templates/registry/registrationConfirmation/message.html.j2 b/packages/dsw-mailer/templates/registry/registrationConfirmation/message.html.j2 index c9358ba4..86ed322a 100644 --- a/packages/dsw-mailer/templates/registry/registrationConfirmation/message.html.j2 +++ b/packages/dsw-mailer/templates/registry/registrationConfirmation/message.html.j2 @@ -17,7 +17,7 @@ -

+

Hello,

Your registration of {{ ctx.organizationName }} ({{ ctx.organizationId }}) is almost done. Just one more step to go. To activate the account, please click on the button below.

@@ -71,9 +71,9 @@ - diff --git a/packages/dsw-mailer/templates/registry/registrationCreatedAnalytics/message.html.j2 b/packages/dsw-mailer/templates/registry/registrationCreatedAnalytics/message.html.j2 index cce8501b..e13fb7ff 100644 --- a/packages/dsw-mailer/templates/registry/registrationCreatedAnalytics/message.html.j2 +++ b/packages/dsw-mailer/templates/registry/registrationCreatedAnalytics/message.html.j2 @@ -15,7 +15,7 @@
-

- Thank you for using the DSW!

+

+

+ Thank you for using DSW!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}

-

+

Hello!

Good news — we have a new organization {% if ctx.appTitle %}in {{ ctx.appTitle }}{% endif %} ({{ ctx.clientUrl }}):

@@ -28,7 +28,7 @@
{{ ctx.organizationEmail }}

-

+

Have a nice day!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}

diff --git a/packages/dsw-mailer/templates/registry/resetToken/message.html.j2 b/packages/dsw-mailer/templates/registry/resetToken/message.html.j2 index e0a698b0..a9d6dd16 100644 --- a/packages/dsw-mailer/templates/registry/resetToken/message.html.j2 +++ b/packages/dsw-mailer/templates/registry/resetToken/message.html.j2 @@ -14,8 +14,8 @@ -
-

+

+

Hello,

Lost the token for {{ ctx.organizationName }} ({{ ctx.organizationId }})? No problem, just click the button below to set up a new one.

@@ -65,9 +65,9 @@ - diff --git a/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.html.j2 b/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.html.j2 index 97c18a5f..6eb35c34 100644 --- a/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.html.j2 +++ b/packages/dsw-mailer/templates/wizard/apiKeyCreated/message.html.j2 @@ -14,8 +14,8 @@
-

- Thank you for using the DSW!

+

+

+ Thank you for using DSW!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}

-
-

+

+

Dear {{ ctx.userFirstName }},

A new API key named {{ ctx.tokenName }} has been created for your account.
It will expire on {{ ctx.tokenExpiresAt|datetime_format("%B %-d, %Y (at %H:%M %Z)") }}. @@ -66,12 +66,12 @@ - diff --git a/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.html.j2 b/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.html.j2 index f68f235d..5c7fa04d 100644 --- a/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.html.j2 +++ b/packages/dsw-mailer/templates/wizard/apiKeyExpiration/message.html.j2 @@ -14,8 +14,8 @@
-

+

+

{% if ctx.supportEmail %} If you did not create this API key or encounter any issues, please contact us via {{ ctx.supportEmail }}. {% endif %}

- Thank you for using the DSW!

+ Thank you for using DSW!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}

-
-

+

+

Dear {{ ctx.userFirstName }},

Your API key, identified as {{ ctx.tokenName }}, is scheduled to expire on {{ ctx.tokenExpiresAt|datetime_format("%B %-d, %Y (at %H:%M %Z)") }}.

@@ -65,12 +65,12 @@ - diff --git a/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.html.j2 b/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.html.j2 new file mode 100644 index 00000000..c4d881e8 --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.html.j2 @@ -0,0 +1,133 @@ +{% extends '_common/layout.html.j2' %} +{% block body %} +{%- set projectGroups = ctx.notifications|sort(attribute='questionnaireName')|groupby('questionnaireUuid') -%} +{%- set spThread = 'threads' if ctx.notifications|length > 1 else 'thread' -%} +{%- macro threadLink(projectUuid, path, threadUuid) -%} +{{ ctx.clientUrl }}/projects/{{ projectUuid }}?questionPath={{ path }}&commentThreadUuid={{ threadUuid }} +{%- endmacro -%} +{%- macro projectLink(projectUuid) -%} +{{ ctx.clientUrl }}/projects/{{ projectUuid }} +{%- endmacro -%} +
-

+

+

{% if ctx.supportEmail %} If you encounter any issues, please contact us via {{ ctx.supportEmail }}. {% endif %}

- Thank you for using the DSW!

+ Thank you for using DSW!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}

+ + + + + +
+ + + + + + + + +
+

+ Hello {{ ctx.userFirstName }},

+ You have been assigned to the following comment {{ spThread }}{% if projectGroups|length > 1 %} within {{ projectGroups|length }} projects{% endif %}: +

+
+ + +
+ +{% for qtnUuid, items in projectGroups %} + + + + + + +
+ + + + + + +
+

+ {{ items[0].questionnaireName }} +

+ {% for item in items %} + + + + + + +
+

+ {{ item.text|no_markdown|truncate(140, False, '...') }} +

+ + + + + + + +
+ Assigned by {{ item.assignedBy.firstName }} {{ item.assignedBy.lastName }} + + Open +
+
+ {% endfor %} +
+
+{% endfor %} + + + + + + +
+ + + + + + + + +
+

+ View all comment threads assigned to you +


+ {% if ctx.supportEmail %} + If any problem occurs contact us via {{ ctx.supportEmail }}.

+ {% endif %} + Thank you for using DSW!

+ {% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %} +

+
+ + +
+{% endblock %} diff --git a/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.json b/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.json new file mode 100644 index 00000000..1b79a144 --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.json @@ -0,0 +1,16 @@ +{ + "id": "wizard:commentThreadAssigned", + "subject": "Assigned to Comment Threads", + "subjectPrefix": true, + "defaultSenderName": "DSW", + "parts": [ + { + "type": "html", + "file": "message.html.j2" + }, + { + "type": "plain", + "file": "message.txt.j2" + } + ] +} diff --git a/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.txt.j2 b/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.txt.j2 new file mode 100644 index 00000000..30a59a9b --- /dev/null +++ b/packages/dsw-mailer/templates/wizard/commentThreadAssigned/message.txt.j2 @@ -0,0 +1,24 @@ +{%- set projectGroups = ctx.notifications|sort(attribute='questionnaireName')|groupby('questionnaireUuid') -%} +{%- set spThread = 'threads' if ctx.notifications|length > 1 else 'thread' -%} +{%- macro threadLink(projectUuid, path, threadUuid) -%} +{{ ctx.clientUrl }}/projects/{{ projectUuid }}?questionPath={{ path }}&commentThreadUuid={{ threadUuid }} +{%- endmacro -%} +{%- macro projectLink(projectUuid) -%} +{{ ctx.clientUrl }}/projects/{{ projectUuid }} +{%- endmacro -%} + +Hello {{ ctx.userFirstName }}, + +You have been assigned to the following comment {{ spThread }}{% if projectGroups|length > 1 %} (within {{ projectGroups|length }} projects){% endif %}: + +{% for qtnUuid, items in projectGroups %} +{{ items[0].questionnaireName }} ({{ projectLink(qtnUuid) }}) +{% for item in items %} +- {{ item.text|truncate(100, False, '...') }} + Assigned by: {{ item.assignedBy.firstName }} {{ item.assignedBy.lastName }}, {% if item.resolved %}already resolved{% else %}unresolved{% endif %}{% if item.private %}, private editor note{% endif %}) + {{ threadLink(item.questionnaireUuid, item.path, item.commentThreadUuid) }} +{% endfor %} +-------------------- +{% endfor %} + +{% include '_common/footer.txt.j2' %} diff --git a/packages/dsw-mailer/templates/wizard/questionnaireInvitation/message.html.j2 b/packages/dsw-mailer/templates/wizard/questionnaireInvitation/message.html.j2 index 540f3769..72c45289 100644 --- a/packages/dsw-mailer/templates/wizard/questionnaireInvitation/message.html.j2 +++ b/packages/dsw-mailer/templates/wizard/questionnaireInvitation/message.html.j2 @@ -16,7 +16,7 @@

+ style="margin: 10px 0; padding: 0; mso-line-height-rule: exactly; -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%; color: #212529; font-family: Helvetica; font-size: 14px; line-height: 150%; text-align: left;"> Hello {{ ctx.inviteeFirstName }},

{{ ctx.ownerFirstName }} {{ ctx.ownerLastName }} has invited you to the project {{ ctx.questionnaireName }}.

@@ -66,12 +66,12 @@ - diff --git a/packages/dsw-mailer/templates/wizard/registrationConfirmation/message.html.j2 b/packages/dsw-mailer/templates/wizard/registrationConfirmation/message.html.j2 index fc7b2725..a3c07cf0 100644 --- a/packages/dsw-mailer/templates/wizard/registrationConfirmation/message.html.j2 +++ b/packages/dsw-mailer/templates/wizard/registrationConfirmation/message.html.j2 @@ -16,7 +16,7 @@
-

+

+

{% if ctx.supportEmail %} If any problem occurs contact us via {{ ctx.supportEmail }}.

{% endif %} - Thank you for using the DSW!

+ Thank you for using DSW!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}

+ style="margin: 10px 0; padding: 0; mso-line-height-rule: exactly; -ms-text-size-adjust: 100%; -webkit-text-size-adjust: 100%; color: #212529; font-family: Helvetica; font-size: 14px; line-height: 150%; text-align: left;"> Hello {{ ctx.userFirstName }},

Welcome aboard! Well you’re almost set. Just one more step and you can start using the DSW. To activate your account, please click on the button below.

@@ -66,12 +66,12 @@ - diff --git a/packages/dsw-mailer/templates/wizard/registrationCreatedAnalytics/message.html.j2 b/packages/dsw-mailer/templates/wizard/registrationCreatedAnalytics/message.html.j2 index a561201a..dbc932cc 100644 --- a/packages/dsw-mailer/templates/wizard/registrationCreatedAnalytics/message.html.j2 +++ b/packages/dsw-mailer/templates/wizard/registrationCreatedAnalytics/message.html.j2 @@ -15,7 +15,7 @@
-

+

+

{% if ctx.supportEmail %} If any problem occurs contact us via {{ ctx.supportEmail }}.

{% endif %} - Thank you for using the DSW!

+ Thank you for using DSW!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}

-

+

Hello!

Good news — we have a new user {% if ctx.appTitle %}in {{ ctx.appTitle }} {% endif %}({{ ctx.clientUrl }}):

@@ -26,7 +26,7 @@
{{ ctx.userEmail }}

-

+

Have a nice day!

Your {{ ctx.appTitle }}

diff --git a/packages/dsw-mailer/templates/wizard/resetPassword/message.html.j2 b/packages/dsw-mailer/templates/wizard/resetPassword/message.html.j2 index 74a4a88c..cd0ca6dd 100644 --- a/packages/dsw-mailer/templates/wizard/resetPassword/message.html.j2 +++ b/packages/dsw-mailer/templates/wizard/resetPassword/message.html.j2 @@ -14,8 +14,8 @@ -
-

+

+

Hi {{ ctx.userFirstName }},

Forgot your password? No problem, that happens to the best of us! You can easily set up a new password by clicking on the button below.

@@ -65,12 +65,12 @@ - diff --git a/packages/dsw-mailer/templates/wizard/twoFactorAuth/message.html.j2 b/packages/dsw-mailer/templates/wizard/twoFactorAuth/message.html.j2 index 38838d67..707c17bb 100644 --- a/packages/dsw-mailer/templates/wizard/twoFactorAuth/message.html.j2 +++ b/packages/dsw-mailer/templates/wizard/twoFactorAuth/message.html.j2 @@ -14,8 +14,8 @@
-

+

+

{% if ctx.supportEmail %} If any problem occurs contact us via {{ ctx.supportEmail }}.

{% endif %} - Thank you for using the DSW!

+ Thank you for using DSW!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}

-
-

+

+

Dear {{ ctx.userFirstName }},

You have requested an authentication code to log into your account. Your code is:

@@ -63,13 +63,13 @@ - diff --git a/packages/dsw-models/CHANGELOG.md b/packages/dsw-models/CHANGELOG.md index ab1f6a71..90b32eb9 100644 --- a/packages/dsw-models/CHANGELOG.md +++ b/packages/dsw-models/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.9.0] + +Released for version consistency with other DSW tools. + ## [4.8.1] Released for version consistency with other DSW tools. @@ -150,3 +154,4 @@ Released for version consistency with other DSW tools. [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-models/pyproject.toml b/packages/dsw-models/pyproject.toml index e21a3fff..8b1b12f1 100644 --- a/packages/dsw-models/pyproject.toml +++ b/packages/dsw-models/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-models' -version = "4.8.1" +version = "4.9.0" description = 'Library with DSW models and basic IO operations' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-storage/CHANGELOG.md b/packages/dsw-storage/CHANGELOG.md index 3f26ebc2..e720c944 100644 --- a/packages/dsw-storage/CHANGELOG.md +++ b/packages/dsw-storage/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.9.0] + +Released for version consistency with other DSW tools. + ## [4.8.1] Released for version consistency with other DSW tools. @@ -217,3 +221,4 @@ Released for version consistency with other DSW tools. [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-storage/pyproject.toml b/packages/dsw-storage/pyproject.toml index eba08405..6e486367 100644 --- a/packages/dsw-storage/pyproject.toml +++ b/packages/dsw-storage/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-storage' -version = "4.8.1" +version = "4.9.0" description = 'Library for managing DSW S3 storage' readme = 'README.md' keywords = ['dsw', 's3', 'bucket', 'storage'] @@ -26,7 +26,7 @@ dependencies = [ 'minio', 'tenacity', # DSW - "dsw-config==4.8.1", + "dsw-config==4.9.0", ] [project.urls] diff --git a/packages/dsw-storage/requirements.txt b/packages/dsw-storage/requirements.txt index 98d9a61b..04471d18 100644 --- a/packages/dsw-storage/requirements.txt +++ b/packages/dsw-storage/requirements.txt @@ -1,6 +1,6 @@ -certifi==2024.6.2 +certifi==2024.7.4 minio==7.2.7 PyYAML==6.0.1 -sentry-sdk==2.6.0 -tenacity==8.4.1 +sentry-sdk==2.8.0 +tenacity==8.5.0 urllib3==2.2.2 diff --git a/packages/dsw-tdk/CHANGELOG.md b/packages/dsw-tdk/CHANGELOG.md index 793f9e3b..e337da70 100644 --- a/packages/dsw-tdk/CHANGELOG.md +++ b/packages/dsw-tdk/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [4.9.0] + +Released for version consistency with other DSW tools. + ## [4.8.1] ### Changed @@ -441,3 +445,4 @@ Initial DSW Template Development Kit (versioned as part of the [DSW platform](ht [4.7.0]: /../../tree/v4.7.0 [4.8.0]: /../../tree/v4.8.0 [4.8.1]: /../../tree/v4.8.1 +[4.9.0]: /../../tree/v4.9.0 diff --git a/packages/dsw-tdk/dsw/tdk/consts.py b/packages/dsw-tdk/dsw/tdk/consts.py index 77d2d64b..2870803d 100644 --- a/packages/dsw-tdk/dsw/tdk/consts.py +++ b/packages/dsw-tdk/dsw/tdk/consts.py @@ -3,7 +3,7 @@ import re APP = 'dsw-tdk' -VERSION = '4.8.1' +VERSION = '4.9.0' METAMODEL_VERSION = 13 REGEX_SEMVER = re.compile(r'^[0-9]+\.[0-9]+\.[0-9]+$') diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index d418ad31..fc6cf00a 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-tdk' -version = "4.8.1" +version = "4.9.0" description = 'Data Stewardship Wizard Template Development Toolkit' readme = 'README.md' keywords = ['documents', 'dsw', 'jinja2', 'template', 'toolkit'] diff --git a/packages/dsw-tdk/requirements.txt b/packages/dsw-tdk/requirements.txt index 684ee669..e4bb518a 100644 --- a/packages/dsw-tdk/requirements.txt +++ b/packages/dsw-tdk/requirements.txt @@ -7,7 +7,7 @@ charset-normalizer==3.3.2 click==8.1.7 colorama==0.4.6 frozenlist==1.4.1 -humanize==4.9.0 +humanize==4.10.0 idna==3.7 Jinja2==3.1.4 MarkupSafe==2.1.5 diff --git a/requirements.txt b/requirements.txt index 20462d85..c9fe6291 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,18 +4,18 @@ anyio==4.4.0 async-timeout==4.0.3 attrs==23.2.0 Brotli==1.1.0 -certifi==2024.6.2 +certifi==2024.7.4 cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 colorama==0.4.6 cssselect2==0.7.0 -dkimpy==1.1.7 +dkimpy==1.1.8 dnspython==2.6.1 -fonttools==4.53.0 +fonttools==4.53.1 frozenlist==1.4.1 html5lib==1.1 -humanize==4.9.0 +humanize==4.10.0 idna==3.7 isodate==0.6.1 Jinja2==3.1.4 @@ -26,9 +26,9 @@ minio==7.2.7 multidict==6.0.5 pathspec==0.12.1 pathvalidate==3.2.0 -Pillow==10.3.0 -psycopg==3.1.19 -psycopg-binary==3.1.19 +Pillow==10.4.0 +psycopg==3.2.1 +psycopg-binary==3.2.1 pycparser==2.22 pydyf==0.10.0 pyparsing==3.1.2 @@ -40,10 +40,10 @@ PyYAML==6.0.1 rdflib==7.0.0 rdflib-jsonld==0.6.2 requests==2.32.3 -sentry-sdk==2.6.0 +sentry-sdk==2.8.0 six==1.16.0 sniffio==1.3.1 -tenacity==8.4.1 +tenacity==8.5.0 text-unidecode==1.3 tinycss2==1.3.0 typing_extensions==4.12.2
-

+

+

This code is valid for {{ ctx.clientUrl }}. {% if ctx.supportEmail %} If you did not initiate this login attempt, please contact us via {{ ctx.supportEmail }}. {% endif %}

- Thank you for using the DSW!

+ Thank you for using DSW!

{% if ctx.appTitle %}{{ ctx.appTitle }} team{% endif %}