Merge pull request #181 from ds-wizard/release/4.4.0

Release 4.4.0

packages/dsw-command-queue/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.4.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.3.1]
 
 Released for version consistency with other DSW tools.
@@ -184,3 +188,4 @@ Released for version consistency with other DSW tools.
 [4.2.1]: /../../tree/v4.2.1
 [4.3.0]: /../../tree/v4.3.0
 [4.3.1]: /../../tree/v4.3.1
+[4.4.0]: /../../tree/v4.4.0

packages/dsw-command-queue/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-command-queue'
-version = "4.3.1"
+version = "4.4.0"
 description = 'Library for working with command queue and persistent commands'
 readme = 'README.md'
 keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing']
@@ -25,7 +25,7 @@ classifiers = [
 requires-python = '>=3.10, <4'
 dependencies = [
     # DSW
-    "dsw-database==4.3.1",
+    "dsw-database==4.4.0",
 ]
 
 [project.urls]

packages/dsw-config/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.4.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.3.1]
 
 Released for version consistency with other DSW tools.
@@ -192,3 +196,4 @@ Released for version consistency with other DSW tools.
 [4.2.1]: /../../tree/v4.2.1
 [4.3.0]: /../../tree/v4.3.0
 [4.3.1]: /../../tree/v4.3.1
+[4.4.0]: /../../tree/v4.4.0

packages/dsw-config/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-config'
-version = "4.3.1"
+version = "4.4.0"
 description = 'Library for DSW config manipulation'
 readme = 'README.md'
 keywords = ['dsw', 'config', 'yaml', 'parser']

packages/dsw-data-seeder/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.4.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.3.1]
 
 Released for version consistency with other DSW tools.
@@ -242,3 +246,4 @@ Released for version consistency with other DSW tools.
 [4.2.1]: /../../tree/v4.2.1
 [4.3.0]: /../../tree/v4.3.0
 [4.3.1]: /../../tree/v4.3.1
+[4.4.0]: /../../tree/v4.4.0

packages/dsw-data-seeder/Dockerfile

@@ -1,4 +1,4 @@
-FROM datastewardshipwizard/python-base:4.3.0-3.11-basic as builder
+FROM datastewardshipwizard/python-base:4.4.0-3.11-basic as builder
 
 WORKDIR /app
 
@@ -15,7 +15,7 @@ RUN python -m pip wheel --no-cache-dir --wheel-dir=/app/wheels -r /app/packages/
  && python -m pip wheel --no-cache-dir --no-deps --wheel-dir=/app/wheels /app/packages/dsw-data-seeder
 
 
-FROM datastewardshipwizard/python-base:4.3.0-rc.1-3.11-basic
+FROM datastewardshipwizard/python-base:4.4.0-3.11-basic
 
 ENV APPLICATION_CONFIG_PATH /app/config/application.yml
 ENV WORKDIR_PATH /home/user/data

packages/dsw-data-seeder/dsw/data_seeder/consts.py

@@ -6,7 +6,7 @@
 DEFAULT_PLACEHOLDER = '<<|TENANT-ID|>>'
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 PROG_NAME = 'dsw-data-seeder'
-VERSION = '4.3.1'
+VERSION = '4.4.0'
 
 VAR_APP_CONFIG_PATH = 'APPLICATION_CONFIG_PATH'
 VAR_WORKDIR_PATH = 'WORKDIR_PATH'

packages/dsw-data-seeder/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-data-seeder'
-version = "4.3.1"
+version = "4.4.0"
 description = 'Worker for seeding DSW data'
 readme = 'README.md'
 keywords = ['data', 'database', 'seed', 'storage']
@@ -29,10 +29,10 @@ dependencies = [
     'sentry-sdk',
     'tenacity',
     # DSW
-    "dsw-command-queue==4.3.1",
-    "dsw-config==4.3.1",
-    "dsw-database==4.3.1",
-    "dsw-storage==4.3.1",
+    "dsw-command-queue==4.4.0",
+    "dsw-config==4.4.0",
+    "dsw-database==4.4.0",
+    "dsw-storage==4.4.0",
 ]
 
 [project.urls]

packages/dsw-database/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.4.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.3.1]
 
 ### Fixed
@@ -201,3 +205,4 @@ Released for version consistency with other DSW tools.
 [4.2.1]: /../../tree/v4.2.1
 [4.3.0]: /../../tree/v4.3.0
 [4.3.1]: /../../tree/v4.3.1
+[4.4.0]: /../../tree/v4.4.0

packages/dsw-database/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-database'
-version = "4.3.1"
+version = "4.4.0"
 description = 'Library for managing DSW database'
 readme = 'README.md'
 keywords = ['dsw', 'database']
@@ -26,7 +26,7 @@ dependencies = [
     'psycopg[binary]',
     'tenacity',
     # DSW
-    "dsw-config==4.3.1",
+    "dsw-config==4.4.0",
 ]
 
 [project.urls]

packages/dsw-document-worker/CHANGELOG.md

@@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.4.0]
+
+### Changed
+
+- Enhanced support for Pandoc filters in `pandoc` step
+
 ## [4.3.1]
 
 Released for version consistency with other DSW tools.
@@ -245,3 +251,4 @@ Released for version consistency with other DSW tools.
 [4.2.1]: /../../tree/v4.2.1
 [4.3.0]: /../../tree/v4.3.0
 [4.3.1]: /../../tree/v4.3.1
+[4.4.0]: /../../tree/v4.4.0

packages/dsw-document-worker/Dockerfile

@@ -1,4 +1,4 @@
-FROM datastewardshipwizard/python-base:4.3.0-3.11-docworker as builder
+FROM datastewardshipwizard/python-base:4.4.0-3.11-docworker as builder
 
 ARG TARGETARCH
 
@@ -17,14 +17,18 @@ RUN python -m pip wheel --wheel-dir=/app/wheels -r /app/packages/dsw-document-wo
  && python -m pip wheel --no-deps --wheel-dir=/app/wheels /app/packages/dsw-document-worker/addons/* \
  && python -m pip wheel --no-deps --wheel-dir=/app/wheels /app/packages/dsw-document-worker
 
-FROM datastewardshipwizard/python-base:4.3.0-3.11-docworker
+FROM datastewardshipwizard/python-base:4.4.0-3.11-docworker
 
 ENV APPLICATION_CONFIG_PATH /app/config/application.yml
 ENV WORKDIR_PATH /tmp/docworker
 ENV PATH "/home/user/.local/bin:$PATH"
 
-# Add fonts and data
-COPY packages/dsw-document-worker/fonts /usr/share/fonts/truetype/custom
+# Add fonts
+COPY packages/dsw-document-worker/resources/fonts /usr/share/fonts/truetype/custom
+RUN fc-cache
+
+# Add Pandoc filters
+COPY packages/dsw-document-worker/resources/pandoc/filters /pandoc/filters
 RUN fc-cache
 
 # Use non-root user

packages/dsw-document-worker/dsw/document_worker/consts.py

@@ -6,7 +6,7 @@
 EXIT_SUCCESS = 0
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 PROG_NAME = 'docworker'
-VERSION = '4.3.1'
+VERSION = '4.4.0'
 
 VAR_APP_CONFIG_PATH = 'APPLICATION_CONFIG_PATH'
 VAR_WORKDIR_PATH = 'WORKDIR_PATH'

packages/dsw-document-worker/dsw/document_worker/conversions.py

@@ -1,4 +1,6 @@
 import logging
+import os
+import pathlib
 import rdflib
 import shlex
 import subprocess
@@ -44,16 +46,43 @@ def __str__(self):
 
 
 class Pandoc:
+    FILTERS_PATH = pathlib.Path(os.getenv('PANDOC_FILTERS', '/pandoc/filters'))
+    TEMPLATES_PATH = pathlib.Path(os.getenv('PANDOC_TEMPLATES', '/pandoc/templates'))
 
-    def __init__(self, config: DocumentWorkerConfig):
+    def __init__(self, config: DocumentWorkerConfig, filter_names: list[str], template_name: str | None):
         self.config = config
+        self.filter_names = filter_names
+        self.template_name = template_name
+        self._check_filters()
+        self._check_template()
+
+    def _check_filters(self):
+        for name in self.filter_names:
+            if not (self.FILTERS_PATH / name).is_file():
+                raise RuntimeError(f'Pandoc filter "{name}" not found')
+
+    def _check_template(self):
+        if self.template_name and not (self.TEMPLATES_PATH / self.template_name).is_file():
+            raise RuntimeError(f'Pandoc template "{self.template_name}" not found')
+
+    def _extra_args(self):
+        args = []
+        if self.template_name:
+            args.extend(['--template', str(self.TEMPLATES_PATH / self.template_name)])
+        for filter_name in self.filter_names:
+            if filter_name.endswith('.lua'):
+                args.extend(['--lua-filter', str(self.FILTERS_PATH / filter_name)])
+            else:
+                args.extend(['--filter', str(self.FILTERS_PATH / filter_name)])
+        return shlex.split(' '.join(args))
 
     def __call__(self, source_format: FileFormat, target_format: FileFormat,
                  data: bytes, metadata: dict, workdir: str) -> bytes:
         args = ['-f', source_format.name, '-t', target_format.name, '-o', '-']
         config_args = shlex.split(self.config.pandoc.args)
         template_args = self.extract_template_args(metadata)
-        command = self.config.pandoc.command + template_args + config_args + args
+        extra_args = self._extra_args()
+        command = self.config.pandoc.command + template_args + config_args + extra_args + args
         return run_conversion(
             args=command,
             workdir=workdir,

packages/dsw-document-worker/dsw/document_worker/templates/steps/conversion.py

@@ -100,16 +100,31 @@ class PandocStep(Step):
 
     OPTION_FROM = 'from'
     OPTION_TO = 'to'
+    OPTION_FILTERS = 'filters'
+    OPTION_TEMPLATE = 'template'
 
     def __init__(self, template, options: dict):
         super().__init__(template, options)
-        self.pandoc = Pandoc(config=Context.get().app.cfg)
         self.input_format = FileFormats.get(options[self.OPTION_FROM])
         self.output_format = FileFormats.get(options[self.OPTION_TO])
         if self.input_format not in self.INPUT_FORMATS:
             self.raise_exc(f'Unknown input format "{self.input_format.name}"')
         if self.output_format not in self.OUTPUT_FORMATS:
             self.raise_exc(f'Unknown output format "{self.output_format.name}"')
+        self.pandoc = Pandoc(
+            config=Context.get().app.cfg,
+            filter_names=self._extract_filter_names(options.get(self.OPTION_FILTERS, '')),
+            template_name=options.get(self.OPTION_TEMPLATE, None)
+        )
+
+    @staticmethod
+    def _extract_filter_names(filters: str) -> list[str]:
+        names = list()
+        for name in filters.split(','):
+            name = name.strip()
+            if name:
+                names.append(name)
+        return names
 
     def execute_first(self, context: dict) -> DocumentFile:
         return self.raise_exc(f'Step "{self.NAME}" cannot be first')

packages/dsw-document-worker/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-document-worker'
-version = "4.3.1"
+version = "4.4.0"
 description = 'Worker for assembling and transforming documents'
 readme = 'README.md'
 keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker']
@@ -40,10 +40,10 @@ dependencies = [
     'weasyprint',
     'XlsxWriter',
     # DSW
-    "dsw-command-queue==4.3.1",
-    "dsw-config==4.3.1",
-    "dsw-database==4.3.1",
-    "dsw-storage==4.3.1",
+    "dsw-command-queue==4.4.0",
+    "dsw-config==4.4.0",
+    "dsw-database==4.4.0",
+    "dsw-storage==4.4.0",
 ]
 
 [project.urls]

packages/dsw-document-worker/resources/pandoc/filters/docx-pagebreak.lua

@@ -0,0 +1,35 @@
+local ooxml = function (s)
+  return pandoc.RawBlock('openxml', s)
+end
+
+local pagebreak = ooxml [[
+  <w:p><w:r><w:br w:type="page" /></w:r></w:p>
+]]
+
+function _Div (div)
+  if div.classes:includes 'newpage-before' then
+    div.content:insert(1, pagebreak)
+  end
+  if div.classes:includes 'newpage-after' then
+    div.content:insert(pagebreak)
+  end
+  return div
+end
+
+function _Para (para)
+  for _, inline in ipairs(para.content) do
+    if inline.t == 'Str' and inline.text == '\\newpage' then
+      return pagebreak
+    end
+  end
+end
+
+function _RawBlock (raw)
+  if raw.text == '\\newpage' then
+    return pagebreak
+  end
+end
+
+if FORMAT == 'docx' then
+    return { { Div = _Div, Plain = _Para, Para = _Para, RawBlock = _RawBlock } }
+end