Merge pull request #218 from ds-wizard/release/4.9.0

Release 4.9.0

.github/workflows/build.yml

@@ -176,6 +176,7 @@ jobs:
       with:
         images: |
           ${{ env.PUBLIC_IMAGE_PREFIX }}/${{ steps.docker-image-name.outputs.NAME }}
+          ${{ secrets.DOCKER_HUB_USERNAME }}/${{ steps.docker-image-name.outputs.NAME }}
         tags: |
           type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
           type=semver,pattern={{version}}

packages/dsw-command-queue/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.9.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.8.1]
 
 Released for version consistency with other DSW tools.
@@ -221,3 +225,4 @@ Released for version consistency with other DSW tools.
 [4.7.0]: /../../tree/v4.7.0
 [4.8.0]: /../../tree/v4.8.0
 [4.8.1]: /../../tree/v4.8.1
+[4.9.0]: /../../tree/v4.9.0

packages/dsw-command-queue/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-command-queue'
-version = "4.8.1"
+version = "4.9.0"
 description = 'Library for working with command queue and persistent commands'
 readme = 'README.md'
 keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing']
@@ -25,7 +25,7 @@ classifiers = [
 requires-python = '>=3.10, <4'
 dependencies = [
     # DSW
-    "dsw-database==4.8.1",
+    "dsw-database==4.9.0",
 ]
 
 [project.urls]

packages/dsw-command-queue/requirements.txt

@@ -1,6 +1,6 @@
-psycopg==3.1.19
-psycopg-binary==3.1.19
+psycopg==3.2.1
+psycopg-binary==3.2.1
 PyYAML==6.0.1
-tenacity==8.4.1
+tenacity==8.5.0
 typing_extensions==4.12.2
 tzdata==2024.1

packages/dsw-config/CHANGELOG.md

@@ -8,6 +8,16 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.9.0]
+
+### Added
+
+- Included AWS-specific configuration that `engine-backend` uses
+
+### Changed
+
+- Moved `mail` configuration to `dsw-mailer` package
+
 ## [4.8.1]
 
 Released for version consistency with other DSW tools.
@@ -227,3 +237,4 @@ Released for version consistency with other DSW tools.
 [4.7.0]: /../../tree/v4.7.0
 [4.8.0]: /../../tree/v4.8.0
 [4.8.1]: /../../tree/v4.8.1
+[4.9.0]: /../../tree/v4.9.0

packages/dsw-config/dsw/config/keys.py

@@ -232,99 +232,30 @@ class _S3Keys(ConfigKeysContainer):
     )
 
 
-class _MailKeys(ConfigKeysContainer):
-    enabled = ConfigKey(
-        yaml_path=['mail', 'enabled'],
-        var_names=['MAIL_ENABLED'],
-        default=True,
-        cast=cast_bool,
-    )
-    name = ConfigKey(
-        yaml_path=['mail', 'name'],
-        var_names=['MAIL_NAME'],
-        default='',
-        cast=cast_str,
-    )
-    email = ConfigKey(
-        yaml_path=['mail', 'email'],
-        var_names=['MAIL_EMAIL'],
-        default='',
-        cast=cast_str,
-    )
-    host = ConfigKey(
-        yaml_path=['mail', 'host'],
-        var_names=['MAIL_HOST'],
-        default='',
-        cast=cast_str,
-    )
-    port = ConfigKey(
-        yaml_path=['mail', 'port'],
-        var_names=['MAIL_PORT'],
-        cast=cast_str,
-    )
-    ssl = ConfigKey(
-        yaml_path=['mail', 'ssl'],
-        var_names=[],
-        cast=cast_optional_str,
-    )
-    security = ConfigKey(
-        yaml_path=['mail', 'security'],
-        var_names=['MAIL_SECURITY'],
+class _AWSKeys(ConfigKeysContainer):
+    access_key_id = ConfigKey(
+        yaml_path=['aws', 'awsAccessKeyId'],
+        var_names=['AWS_AWS_ACCESS_KEY_ID'],
         cast=cast_optional_str,
     )
-    auth_enabled = ConfigKey(
-        yaml_path=['mail', 'authEnabled'],
-        var_names=[],
-        cast=cast_optional_bool,
-    )
-    username = ConfigKey(
-        yaml_path=['mail', 'username'],
-        var_names=['MAIL_USERNAME'],
+    secret_access_key = ConfigKey(
+        yaml_path=['aws', 'awsSecretAccessKey'],
+        var_names=['AWS_AWS_SECRET_ACCESS_KEY'],
         cast=cast_optional_str,
     )
-    password = ConfigKey(
-        yaml_path=['mail', 'password'],
-        var_names=['MAIL_PASSWORD'],
-        cast=cast_optional_str,
-    )
-    rate_limit_window = ConfigKey(
-        yaml_path=['mail', 'rateLimit', 'window'],
-        var_names=['MAIL_RATE_LIMIT_WINDOW'],
-        default=0,
-        cast=cast_int,
-    )
-    rate_limit_count = ConfigKey(
-        yaml_path=['mail', 'rateLimit', 'count'],
-        var_names=['MAIL_RATE_LIMIT_COUNT'],
-        default=0,
-        cast=cast_int,
-    )
-    timeout = ConfigKey(
-        yaml_path=['mail', 'timeout'],
-        var_names=['MAIL_TIMEOUT'],
-        default=10,
-        cast=cast_int,
-    )
-    dkim_selector = ConfigKey(
-        yaml_path=['mail', 'dkim', 'selector'],
-        var_names=['MAIL_DKIM_SELECTOR'],
-        default=None,
-        cast=cast_optional_str,
-    )
-    dkim_privkey_file = ConfigKey(
-        yaml_path=['mail', 'dkim', 'privkey_file'],
-        var_names=['MAIL_DKIM_PRIVKEY_FILE'],
-        default=None,
+    region = ConfigKey(
+        yaml_path=['aws', 'awsRegion'],
+        var_names=['AWS_AWS_REGION'],
         cast=cast_optional_str,
     )
 
 
 class ConfigKeys(ConfigKeysContainer):
+    aws = _AWSKeys
     cloud = _CloudKeys
     database = _DatabaseKeys
     general = _GeneralKeys
     logging = _LoggingKeys
-    mail = _MailKeys
     s3 = _S3Keys
     sentry = _SentryKeys
 

packages/dsw-config/dsw/config/model.py

@@ -1,4 +1,3 @@
-import pathlib
 from typing import Optional
 
 from .logging import prepare_logging, LOG_FILTER
@@ -74,93 +73,20 @@ def set_logging_extra(key: str, value: str):
         LOG_FILTER.set_extra(key, value)
 
 
-class CloudConfig(ConfigModel):
-
-    def __init__(self, multi_tenant: bool):
-        self.multi_tenant = multi_tenant
-
-
-class MailConfig(ConfigModel):
-
-    def __init__(self, enabled: bool, ssl: Optional[bool], name: str, email: str,
-                 host: str, port: Optional[int], security: Optional[str],
-                 auth_enabled: Optional[bool], username: Optional[str],
-                 password: Optional[str], rate_limit_window: int,
-                 rate_limit_count: int, timeout: int,
-                 dkim_selector: Optional[str] = None,
-                 dkim_privkey_file: Optional[str] = None):
-        self.enabled = enabled
-        self.name = name
-        self.email = email
-        self.host = host
-        self.security = 'plain'
-        if security is not None:
-            self.security = security.lower()
-        elif ssl is not None:
-            self.security = 'ssl' if ssl else 'plain'
-        self.port = port or self._default_port()
-        self.auth = auth_enabled
-        if self.auth is None:
-            self.auth = username is not None and password is not None
-        self.username = username
-        self.password = password
-        self.rate_limit_window = rate_limit_window
-        self.rate_limit_count = rate_limit_count
-        self.timeout = timeout
-        self.dkim_selector = dkim_selector
-        self.dkim_privkey_file = dkim_privkey_file
-        self.dkim_privkey = b''
-
-    def load_dkim_privkey(self):
-        if self.dkim_privkey_file is not None:
-            self.dkim_privkey = pathlib.Path(self.dkim_privkey_file).read_bytes()
-            self.dkim_privkey = self.dkim_privkey.replace(b'\r\n', b'\n')
-
-    @property
-    def use_dkim(self):
-        return self.dkim_selector is not None and len(self.dkim_privkey) > 0
-
-    @property
-    def login_user(self) -> str:
-        return self.username or ''
-
-    @property
-    def login_password(self) -> str:
-        return self.password or ''
+class AWSConfig(ConfigModel):
 
-    @property
-    def is_plain(self):
-        return self.security == 'plain'
-
-    @property
-    def is_ssl(self):
-        return self.security == 'ssl'
+    def __init__(self, access_key_id: Optional[str], secret_access_key: Optional[str],
+                 region: Optional[str]):
+        self.access_key_id = access_key_id
+        self.secret_access_key = secret_access_key
+        self.region = region
 
     @property
-    def is_tls(self):
-        return self.security == 'starttls' or self.security == 'tls'
+    def has_credentials(self) -> bool:
+        return self.access_key_id is not None and self.secret_access_key is not None
 
-    def _default_port(self) -> int:
-        if self.is_plain:
-            return 25
-        if self.is_ssl:
-            return 465
-        return 587
 
-    def has_credentials(self) -> bool:
-        return self.username is not None and self.password is not None
+class CloudConfig(ConfigModel):
 
-    def __str__(self):
-        return f'MailConfig\n' \
-               f'- enabled = {self.enabled}\n' \
-               f'- name = {self.name}\n' \
-               f'- email = {self.email}\n' \
-               f'- host = {self.host}\n' \
-               f'- port = {self.port}\n' \
-               f'- security = {self.security}\n' \
-               f'- auth = {self.auth}\n' \
-               f'- rate_limit_window = {self.rate_limit_window}\n' \
-               f'- rate_limit_count = {self.rate_limit_count}\n' \
-               f'- timeout = {self.timeout}\n' \
-               f'- dkim_selector = {self.dkim_selector}\n' \
-               f'- dkim_privkey_file = {self.dkim_privkey_file}\n'
+    def __init__(self, multi_tenant: bool):
+        self.multi_tenant = multi_tenant

packages/dsw-config/dsw/config/parser.py

@@ -5,7 +5,7 @@
 
 from .keys import ConfigKey, ConfigKeys
 from .model import GeneralConfig, SentryConfig, S3Config, \
-    DatabaseConfig, LoggingConfig, CloudConfig, MailConfig
+    DatabaseConfig, LoggingConfig, CloudConfig, AWSConfig
 
 
 class MissingConfigurationError(Exception):
@@ -129,21 +129,9 @@ def general(self) -> GeneralConfig:
         )
 
     @property
-    def mail(self):
-        return MailConfig(
-            enabled=self.get(self.keys.mail.enabled),
-            name=self.get(self.keys.mail.name),
-            email=self.get(self.keys.mail.email),
-            host=self.get(self.keys.mail.host),
-            ssl=self.get(self.keys.mail.ssl),
-            port=self.get(self.keys.mail.port),
-            security=self.get(self.keys.mail.security),
-            auth_enabled=self.get(self.keys.mail.auth_enabled),
-            username=self.get(self.keys.mail.username),
-            password=self.get(self.keys.mail.password),
-            rate_limit_window=int(self.get(self.keys.mail.rate_limit_window)),
-            rate_limit_count=int(self.get(self.keys.mail.rate_limit_count)),
-            timeout=int(self.get(self.keys.mail.timeout)),
-            dkim_selector=self.get(self.keys.mail.dkim_selector),
-            dkim_privkey_file=self.get(self.keys.mail.dkim_privkey_file),
+    def aws(self) -> AWSConfig:
+        return AWSConfig(
+            access_key_id=self.get(self.keys.aws.access_key_id),
+            secret_access_key=self.get(self.keys.aws.secret_access_key),
+            region=self.get(self.keys.aws.region),
         )

packages/dsw-config/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-config'
-version = "4.8.1"
+version = "4.9.0"
 description = 'Library for DSW config manipulation'
 readme = 'README.md'
 keywords = ['dsw', 'config', 'yaml', 'parser']

packages/dsw-config/requirements.txt

@@ -1,4 +1,4 @@
-certifi==2024.6.2
+certifi==2024.7.4
 PyYAML==6.0.1
-sentry-sdk==2.6.0
+sentry-sdk==2.8.0
 urllib3==2.2.2

packages/dsw-data-seeder/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.9.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.8.1]
 
 Released for version consistency with other DSW tools.
@@ -277,3 +281,4 @@ Released for version consistency with other DSW tools.
 [4.7.0]: /../../tree/v4.7.0
 [4.8.0]: /../../tree/v4.8.0
 [4.8.1]: /../../tree/v4.8.1
+[4.9.0]: /../../tree/v4.9.0

packages/dsw-data-seeder/dsw/data_seeder/consts.py

@@ -6,7 +6,7 @@
 DEFAULT_PLACEHOLDER = '<<|TENANT-ID|>>'
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 PROG_NAME = 'dsw-data-seeder'
-VERSION = '4.8.1'
+VERSION = '4.9.0'
 
 VAR_APP_CONFIG_PATH = 'APPLICATION_CONFIG_PATH'
 VAR_WORKDIR_PATH = 'WORKDIR_PATH'

packages/dsw-data-seeder/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-data-seeder'
-version = "4.8.1"
+version = "4.9.0"
 description = 'Worker for seeding DSW data'
 readme = 'README.md'
 keywords = ['data', 'database', 'seed', 'storage']
@@ -29,10 +29,10 @@ dependencies = [
     'sentry-sdk',
     'tenacity',
     # DSW
-    "dsw-command-queue==4.8.1",
-    "dsw-config==4.8.1",
-    "dsw-database==4.8.1",
-    "dsw-storage==4.8.1",
+    "dsw-command-queue==4.9.0",
+    "dsw-config==4.9.0",
+    "dsw-database==4.9.0",
+    "dsw-storage==4.9.0",
 ]
 
 [project.urls]