Allow indented docsub blocks in markdown (#77)

NEWS.d/64.added.md

@@ -0,0 +1 @@
+Support indented substitution blocks in markdown

README.md

@@ -255,6 +255,22 @@ For nested blocks, only top level substitution is performed. Use block `#identif
 <!-- docsub: end #top -->
 ```
 
+When substitution block is indented, the indentation is preserved:
+
+<!-- docsub: begin #top -->
+<!-- docsub: include tests/test_readme_indent/__result__.md -->
+<!-- docsub: lines after 1 upto -1 -->
+```markdown
+* List item
+    <!-- docsub: begin -->
+    <!-- docsub: include sublist.md -->
+    * Sub-item 1
+    * Sub-item 2
+    * Sub-item 3
+    <!-- docsub: end -->
+```
+<!-- docsub: end #top -->
+
 # Directives
 
 * *Block delimiters*: `begin`, `end`

compose.yml

@@ -5,10 +5,12 @@ services:
     working_dir: /work
     entrypoint: tox
     volumes:
+      - mypy_cache:/work/.mypy_cache
       - pip_cache:/root/.cache/pip
       - virtualenv_app_data:/root/.local/share/virtualenv
       - .:/work
 
 volumes:
+  mypy_cache:
   pip_cache:
   virtualenv_app_data:

src/docsub/__base__.py

@@ -69,6 +69,7 @@ class Substitution(SyntaxElement, ABC):
     """
 
     id: Optional[str] = None
+    indent: str = ''
     producers: list['Producer'] = field(default_factory=list)
     modifiers: list['Modifier'] = field(default_factory=list)
 
@@ -95,6 +96,9 @@ def append_command(self, cmd: 'Command') -> None:
     def error_invalid(cls, value: str, loc: Location) -> 'InvalidSubstitution':
         return InvalidSubstitution(f'Invalid docsub substitution: {value}', loc=loc)
 
+    def error_indent(self, loc: Location) -> 'DocsubIndentationError':
+        return DocsubIndentationError('Unexpected indentation', loc=loc)
+
     # processing
 
     def process_content_line(self, line: Line) -> None:
@@ -103,13 +107,25 @@ def process_content_line(self, line: Line) -> None:
 
     def produce_lines(self) -> Iterable[Line]:
         for mod_cmd in self.modifiers:
-            yield from mod_cmd.before_producers(self)
+            yield from self._indent_all(mod_cmd.before_producers(self))
         for prod_cmd in self.producers:
             for line in prod_cmd.produce(self):
-                yield from self._modified_lines(line)
+                yield from self._indent_all(self._modified_lines(line))
         for mod_cmd in self.modifiers:
             yield from mod_cmd.after_producers(self)
 
+    def _indent_all(self, lines: Iterable[Line]) -> Iterable[Line]:
+        if not self.indent:
+            yield from lines
+        else:
+            for line in lines:
+                yield self._indent_one(line)
+
+    def _indent_one(self, line: Line) -> Line:
+        if self.indent:
+            line.text = f'{self.indent}{line.text}'
+        return line
+
     def _modified_lines(self, line: Line) -> Iterable[Line]:
         lines = (line,)  # type: tuple[Line, ...]
         for cmd in self.modifiers:
@@ -225,6 +241,12 @@ class DocsubfileNotFound(DocsubfileError, FileNotFoundError):
     """
 
 
+class DocsubIndentationError(DocsubError, IndentationError):
+    """
+    Invalid indentation of docsub statement.
+    """
+
+
 class InvalidCommand(DocsubError):
     """
     Invalid docsub command statement.

src/docsub/processors/md.py

@@ -20,7 +20,7 @@
 
 RX_FENCE = re.compile(r'^(?P<indent>\s*)(?P<fence>```+|~~~+).*$')
 
-DOCSUB_PREFIX = r'^\s*<!--\s*docsub:'
+DOCSUB_PREFIX = r'^(?P<indent>\s*)<!--\s*docsub:'
 RX_DOCSUB = re.compile(DOCSUB_PREFIX)
 RX_BEGIN = re.compile(DOCSUB_PREFIX + r'\s*begin(?:\s+#(?P<id>\S+))?\s*-->\s*$')
 RX_END = re.compile(DOCSUB_PREFIX + r'\s*end(?:\s+#(?P<id>\S+))?\s*-->\s*$')
@@ -43,7 +43,12 @@ def match(cls, line: Line) -> Optional[Self]:
             return None
         if not (match := RX_BEGIN.match(line.text)):
             raise cls.error_invalid(line.text, loc=line.loc)
-        return cls(loc=line.loc, id=match.group('id') or None, env=None)
+        return cls(
+            loc=line.loc,
+            id=match.group('id') or None,
+            indent=match.group('indent'),
+            env=None,
+        )
 
     def set_env(self, env: Environment) -> None:
         self.env = env
@@ -55,6 +60,7 @@ def consume_line(self, line: Line) -> Iterable[Line]:
 
         # block end?
         if m := RX_END.match(line.text):
+            self.assert_docsub_indent(m, line)
             if (m.group('id') or None) == self.id:  # end of this block
                 self.validate_assumptions()
                 yield from self.produce_lines()
@@ -67,11 +73,13 @@ def consume_line(self, line: Line) -> Iterable[Line]:
 
         # plain line because all commands consumed?
         if self.all_commands_consumed:
+            self.assert_line_indent(line)
             self.process_content_line(line)
             return
 
         # command?
         if m := RX_CMD.match(line.text):
+            self.assert_docsub_indent(m, line)
             name = m.group('name')
             conf = getattr(self.env.conf.cmd, name, None)
             cmd = COMMANDS[name](
@@ -85,11 +93,20 @@ def consume_line(self, line: Line) -> Iterable[Line]:
             return
 
         # plain line, first after commands
+        self.assert_line_indent(line)
         self.all_commands_consumed = True
         self.validate_assumptions()
         self.process_content_line(line)
         return
 
+    def assert_docsub_indent(self, match: re.Match[str], line: Line) -> None:
+        if not match.group('indent').startswith(self.indent):
+            raise self.error_indent(loc=line.loc)
+
+    def assert_line_indent(self, line: Line) -> None:
+        if not line.text.startswith(self.indent):
+            raise self.error_indent(loc=line.loc)
+
     def validate_assumptions(self) -> None:
         """
         Validate block assumptions.

tests/test_readme_indent.py

@@ -0,0 +1,10 @@
+from subprocess import check_output
+
+
+def test_readme_indent(data_path, python, monkeypatch):
+    monkeypatch.chdir(data_path)
+    result = check_output(
+        args=[python, '-m', 'docsub', 'apply', '__input__.md'],
+        text=True,
+    )
+    assert result == (data_path / '__result__.md').read_text()

tests/test_readme_indent/__input__.md

@@ -0,0 +1,4 @@
+* List item
+    <!-- docsub: begin -->
+    <!-- docsub: include sublist.md -->
+    <!-- docsub: end -->

tests/test_readme_indent/__result__.md

@@ -0,0 +1,7 @@
+* List item
+    <!-- docsub: begin -->
+    <!-- docsub: include sublist.md -->
+    * Sub-item 1
+    * Sub-item 2
+    * Sub-item 3
+    <!-- docsub: end -->

tests/test_readme_indent/sublist.md

@@ -0,0 +1,3 @@
+* Sub-item 1
+* Sub-item 2
+* Sub-item 3