From a47231a090e923947514f6f019cbfef72dbe5f32 Mon Sep 17 00:00:00 2001 From: nichmor Date: Thu, 1 Aug 2024 10:49:08 +0300 Subject: [PATCH] feat: move modules around by refactoring --- src/rattler_build_conda_compat/jinja.py | 121 -------------- .../jinja/__init__.py | 0 src/rattler_build_conda_compat/jinja/jinja.py | 58 +++++++ .../jinja/objects.py | 99 ++++++++++++ tests/__snapshots__/test_jinja.ambr | 144 +++++++++++++++-- tests/data/context.yaml | 14 -- tests/data/mamba_recipe.yaml | 149 ++++++++++++++++++ tests/test_jinja.py | 2 +- 8 files changed, 441 insertions(+), 146 deletions(-) delete mode 100644 src/rattler_build_conda_compat/jinja.py create mode 100644 src/rattler_build_conda_compat/jinja/__init__.py create mode 100644 src/rattler_build_conda_compat/jinja/jinja.py create mode 100644 src/rattler_build_conda_compat/jinja/objects.py delete mode 100644 tests/data/context.yaml create mode 100644 tests/data/mamba_recipe.yaml diff --git a/src/rattler_build_conda_compat/jinja.py b/src/rattler_build_conda_compat/jinja.py deleted file mode 100644 index 7276dce..0000000 --- a/src/rattler_build_conda_compat/jinja.py +++ /dev/null @@ -1,121 +0,0 @@ -from __future__ import annotations - -from typing import Any, TypedDict - -import jinja2 -import yaml -from jinja2 import DebugUndefined - -from rattler_build_conda_compat.loader import load_yaml - - -class RecipeWithContext(TypedDict, total=False): - context: dict[str, str] - - -class _MissingUndefined(DebugUndefined): - def __str__(self) -> str: - """ - By default, `DebugUndefined` return values in the form `{{ value }}`. - `rattler-build` has a different syntax, so we need to override this method, - and return the value in the form `${{ value }}`. - """ - return f"${super().__str__()}" - - -def jinja_env() -> jinja2.Environment: - """ - Create a `rattler-build` specific Jinja2 environment with modified syntax. - """ - - def stub_compatible_pin(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 - return f"compatible_pin {args[0]}" - - def stub_subpackage_pin(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 - return f"subpackage_pin {args[0]}" - - def version_to_build_string(some_string: str) -> str: - """Converts some version by removing the . character and returning only the first two elements of the version)""" - # We first split the string by whitespace and take the first part - split = some_string.split()[0] if some_string.split() else some_string - # We then split the string by . and take the first two parts - parts = split.split(".") - major = parts[0] if len(parts) > 0 else "" - minor = parts[1] if len(parts) > 1 else "" - return f"{major}{minor}" - - def split_filter(s: str, sep: str = " ") -> list[str]: - """Filter that split a string by a separator""" - return s.split(sep) - - env = jinja2.Environment( - variable_start_string="${{", - variable_end_string="}}", - trim_blocks=True, - lstrip_blocks=True, - autoescape=True, - undefined=_MissingUndefined, - ) - - # inject rattler-build recipe functions in jinja environment - env.globals.update( - { - "compiler": lambda x: x + "_compiler_stub", - "stdlib": lambda x: x + "_stdlib_stub", - "pin_subpackage": stub_subpackage_pin, - "pin_compatible": stub_compatible_pin, - "cdt": lambda *args, **kwargs: "cdt_stub", # noqa: ARG005 - } - ) - - # inject rattler-build recipe filters in jinja environment - env.filters.update( - { - "version_to_buildstring": version_to_build_string, - "split": split_filter, - } - ) - return env - - -def load_recipe_context(context: dict[str, str], jinja_env: jinja2.Environment) -> dict[str, str]: - """ - Load all string values from the context dictionary as Jinja2 templates. - """ - # Process each key-value pair in the dictionary - for key, value in context.items(): - # If the value is a string, render it as a template - if isinstance(value, str): - template = jinja_env.from_string(value) - rendered_value = template.render(context) - context[key] = rendered_value - - return context - - -def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, Any]: - """ - Render the recipe using known values from context section. - Unknown values are not evaluated and are kept as it is. - - Examples: - --- - ```python - >>> from pathlib import Path - >>> from rattler_build_conda_compat.loader import load_yaml - >>> recipe_content = load_yaml((Path().resolve() / "tests" / "data" / "eval_recipe_using_context.yaml").read_text()) - >>> evaluated_context = render_recipe_with_context(recipe_content) - >>> assert "my_value-${{ not_present_value }}" == evaluated_context["build"]["string"] - >>> - ``` - """ - env = jinja_env() - context = recipe_content.get("context", {}) - # load all context templates - context_templates = load_recipe_context(context, env) - - # render the rest of the document with the values from the context - # and keep undefined expressions _as is_. - template = env.from_string(yaml.dump(recipe_content)) - rendered_content = template.render(context_templates) - return load_yaml(rendered_content) diff --git a/src/rattler_build_conda_compat/jinja/__init__.py b/src/rattler_build_conda_compat/jinja/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/rattler_build_conda_compat/jinja/jinja.py b/src/rattler_build_conda_compat/jinja/jinja.py new file mode 100644 index 0000000..b7fb052 --- /dev/null +++ b/src/rattler_build_conda_compat/jinja/jinja.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, TypedDict + +if TYPE_CHECKING: + import jinja2 + +import yaml + +from rattler_build_conda_compat.jinja.objects import jinja_env +from rattler_build_conda_compat.loader import load_yaml + + +class RecipeWithContext(TypedDict, total=False): + context: dict[str, str] + + +def load_recipe_context(context: dict[str, str], jinja_env: jinja2.Environment) -> dict[str, str]: + """ + Load all string values from the context dictionary as Jinja2 templates. + """ + # Process each key-value pair in the dictionary + for key, value in context.items(): + # If the value is a string, render it as a template + if isinstance(value, str): + template = jinja_env.from_string(value) + rendered_value = template.render(context) + context[key] = rendered_value + + return context + + +def render_recipe_with_context(recipe_content: RecipeWithContext) -> dict[str, Any]: + """ + Render the recipe using known values from context section. + Unknown values are not evaluated and are kept as it is. + + Examples: + --- + ```python + >>> from pathlib import Path + >>> from rattler_build_conda_compat.loader import load_yaml + >>> recipe_content = load_yaml((Path().resolve() / "tests" / "data" / "eval_recipe_using_context.yaml").read_text()) + >>> evaluated_context = render_recipe_with_context(recipe_content) + >>> assert "my_value-${{ not_present_value }}" == evaluated_context["build"]["string"] + >>> + ``` + """ + env = jinja_env() + context = recipe_content.get("context", {}) + # load all context templates + context_templates = load_recipe_context(context, env) + + # render the rest of the document with the values from the context + # and keep undefined expressions _as is_. + template = env.from_string(yaml.dump(recipe_content)) + rendered_content = template.render(context_templates) + return load_yaml(rendered_content) diff --git a/src/rattler_build_conda_compat/jinja/objects.py b/src/rattler_build_conda_compat/jinja/objects.py new file mode 100644 index 0000000..36d826b --- /dev/null +++ b/src/rattler_build_conda_compat/jinja/objects.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import os + +import jinja2 +from jinja2 import DebugUndefined + + +class _MissingUndefined(DebugUndefined): + def __str__(self) -> str: + """ + By default, `DebugUndefined` return values in the form `{{ value }}`. + `rattler-build` has a different syntax, so we need to override this method, + and return the value in the form `${{ value }}`. + """ + return f"${super().__str__()}" + + +class _Env: + """A class to represent the env object used in rattler-build recipe.""" + + def get(self, env_var: str, default: str | None) -> str: + try: + return str(os.environ[env_var]) + except KeyError: + if default: + return default + return env_var + + def exists(self, env_var: str) -> bool: + return env_var in os.environ + + +def _stub_compatible_pin(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 + return f"compatible_pin {args[0]}" + + +def _stub_subpackage_pin(*args, **kwargs) -> str: # noqa: ARG001, ANN003, ANN002 + return f"subpackage_pin {args[0]}" + + +def _version_to_build_string(some_string: str | _MissingUndefined) -> str: + """ + Converts some version by removing the . character and returning only the first two elements of the version. + If piped value is undefined, it returns the undefined value as is. + """ + if isinstance(some_string, _MissingUndefined): + inner_value = f"{some_string._undefined_name} | version_to_build_string" # noqa: SLF001 + return f"${{{{ {inner_value} }}}}" + # We first split the string by whitespace and take the first part + split = some_string.split()[0] if some_string.split() else some_string + # We then split the string by . and take the first two parts + parts = split.split(".") + major = parts[0] if len(parts) > 0 else "" + minor = parts[1] if len(parts) > 1 else "" + return f"{major}{minor}" + + +def _split_filter(s: str, sep: str = " ") -> list[str]: + """Filter that split a string by a separator""" + return s.split(sep) + + +def jinja_env() -> jinja2.Environment: + """ + Create a `rattler-build` specific Jinja2 environment with modified syntax. + """ + + env = jinja2.Environment( + variable_start_string="${{", + variable_end_string="}}", + trim_blocks=True, + lstrip_blocks=True, + autoescape=True, + undefined=_MissingUndefined, + ) + + env_obj = _Env() + + # inject rattler-build recipe functions in jinja environment + env.globals.update( + { + "compiler": lambda x: x + "_compiler_stub", + "stdlib": lambda x: x + "_stdlib_stub", + "pin_subpackage": _stub_subpackage_pin, + "pin_compatible": _stub_compatible_pin, + "cdt": lambda *args, **kwargs: "cdt_stub", # noqa: ARG005 + "env": env_obj, + } + ) + + # inject rattler-build recipe filters in jinja environment + env.filters.update( + { + "version_to_buildstring": _version_to_build_string, + "split": _split_filter, + } + ) + return env diff --git a/tests/__snapshots__/test_jinja.ambr b/tests/__snapshots__/test_jinja.ambr index 033e509..7833ef9 100644 --- a/tests/__snapshots__/test_jinja.ambr +++ b/tests/__snapshots__/test_jinja.ambr @@ -1,18 +1,142 @@ # serializer version: 1 # name: test_render_recipe_with_context ''' + about: + description: '# Mamba, the Fast Cross-Platform Package Manager + + MY_ENV_VAR + + default_value + + ' + homepage: https://github.com/mamba-org/mamba + license: BSD-3-Clause + license_family: BSD + license_file: LICENSE + repository: https://github.com/mamba-org/mamba + summary: A fast drop-in alternative to conda, using libsolv for dependency resolution build: - string: ${{ blas_variant }}${{ hash }}_foo-1.2.3 + number: '2' context: - cuda_version: ${{ hash }}_cuda_12 - name: foo - name_version: foo-1.2.3 - split_version: 1;2;3 - version: 1.2.3 - package: - func_result: c_compiler_stub - name: foo - version: 1.2.3 + build_number: '2' + libmamba_version: 1.5.8 + libmambapy_version: 1.5.8 + mamba_version: 1.5.8 + name: mamba + release: 2024.03.25 + outputs: + - build: + script: + - '' + - '' + package: + name: libmamba + version: 1.5.8 + requirements: + build: + - cxx_compiler_stub + - cmake + - ninja + - '' + host: + - libsolv >=0.7.23 + - libcurl >=8.4.0 + - fmt + - '' + ignore_run_exports: + by_name: + - spdlog + - python + run: + - libsolv >=0.7.23 + run_exports: + - subpackage_pin libmamba + tests: + - script: + - else: + - if not exist %LIBRARY_PREFIX%\include\mamba\version.hpp (exit 1) + if: unix + then: + - test -d ${PREFIX}/include/mamba + - build: + script: + - '' + - '' + package: + name: libmambapy + version: 1.5.8 + requirements: + build: + - cxx_compiler_stub + - cmake + - ninja + - if: build_platform != target_platform + then: + - python + - cross-python_${{ target_platform }} + - pybind11 + - pybind11-abi + host: + - python + - nlohmann_json + - subpackage_pin libmamba + ignore_run_exports: + by_name: + - spdlog + run: + - python + - subpackage_pin libmamba + run_exports: + - subpackage_pin libmambapy + tests: + - python: + imports: + - libmambapy + - libmambapy.bindings + - script: + - python -c "import libmambapy._version; assert libmambapy._version.__version__ + == '1.5.8'" + - build: + python: + entry_points: + - mamba = mamba.mamba:main + script: + - '' + - '' + string: ${{ python | version_to_build_string }} + package: + name: mamba + version: 1.5.8 + requirements: + build: + - if: build_platform != target_platform + then: + - python + - cross-python_${{ target_platform }} + run: + - python + - conda >=23.9,<24 + - subpackage_pin libmambapy + tests: + - python: + imports: + - mamba + - script: + - mamba --help + - python -c "import mamba._version; assert mamba._version.__version__ == '1.5.8'" + - if: linux + then: + - test -f ${PREFIX}/etc/profile.d/mamba.sh + - mamba create -n test_py2 python=2.7 --dry-run + - mamba install xtensor xsimd -c conda-forge --dry-run + - if: unix + then: + - test -f ${PREFIX}/condabin/mamba + recipe: + name: mamba-split + source: + sha256: 6ddaf4b0758eb7ca1250f427bc40c2c3ede43257a60bac54e4320a4de66759a6 + url: https://github.com/mamba-org/mamba/archive/refs/tags/2024.03.25.tar.gz ''' # --- diff --git a/tests/data/context.yaml b/tests/data/context.yaml deleted file mode 100644 index 6e980d0..0000000 --- a/tests/data/context.yaml +++ /dev/null @@ -1,14 +0,0 @@ -context: - name: "foo" - version: "1.2.3" - name_version: ${{ name }}-${{ version }} - cuda_version: ${{ hash }}_cuda_${{ version | version_to_buildstring }} - split_version: ${{ version | split('.') | join(';') }} - -package: - name: ${{ name }} - version: ${{ version }} - func_result: ${{ compiler('c') }} - -build: - string: ${{ blas_variant }}${{ hash }}_${{ name_version }} diff --git a/tests/data/mamba_recipe.yaml b/tests/data/mamba_recipe.yaml new file mode 100644 index 0000000..d0eb4a2 --- /dev/null +++ b/tests/data/mamba_recipe.yaml @@ -0,0 +1,149 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json + +context: + name: mamba + libmamba_version: "1.5.8" + libmambapy_version: "1.5.8" + mamba_version: "1.5.8" + release: "2024.03.25" + build_number: 2 + +recipe: + name: mamba-split + +source: + url: https://github.com/mamba-org/mamba/archive/refs/tags/${{ release }}.tar.gz + sha256: 6ddaf4b0758eb7ca1250f427bc40c2c3ede43257a60bac54e4320a4de66759a6 + +build: + number: ${{ build_number }} + +outputs: + - package: + name: libmamba + version: ${{ libmamba_version }} + build: + script: + - ${{ "build_mamba.sh" if unix }} + - ${{ "build_mamba.bat" if win }} + requirements: + build: + - ${{ compiler('cxx') }} + - cmake + - ninja + - ${{ "python" if win }} + host: + - libsolv >=0.7.23 + - libcurl >=8.4.0 + - fmt + - ${{ "winreg" if win }} + run: + - libsolv >=0.7.23 + run_exports: + - ${{ pin_subpackage('libmamba', max_pin='x.x') }} + ignore_run_exports: + by_name: + - spdlog + - python + tests: + - script: + - if: unix + then: + - test -d ${PREFIX}/include/mamba # [unix] + else: + - if not exist %LIBRARY_PREFIX%\include\mamba\version.hpp (exit 1) # [win] + + - package: + name: libmambapy + version: ${{ libmambapy_version }} + build: + script: + - ${{ "build_mamba.sh" if unix }} + - ${{ "build_mamba.bat" if win }} + # string: py_sup${{ python | version_to_buildstring }}h${{ hash }}_${{ build_number }} + requirements: + build: + - ${{ compiler('cxx') }} + - cmake + - ninja + - if: build_platform != target_platform + then: + - python + - cross-python_${{ target_platform }} + - pybind11 + - pybind11-abi + host: + - python + - nlohmann_json + - ${{ pin_subpackage('libmamba', exact=True) }} + run: + - python + - ${{ pin_subpackage('libmamba', exact=True) }} + run_exports: + - ${{ pin_subpackage('libmambapy', max_pin='x.x') }} + ignore_run_exports: + by_name: + - spdlog + tests: + - python: + imports: + - libmambapy + - libmambapy.bindings + - script: + - python -c "import libmambapy._version; assert libmambapy._version.__version__ == '${{ libmambapy_version }}'" + + - package: + name: mamba + version: ${{ mamba_version }} + build: + script: + - ${{ "build_mamba.sh" if unix }} + - ${{ "build_mamba.bat" if win }} + # string: py${{ python | version_to_buildstring }}h${{ hash }}_${{ build_number }} + string: ${{ python | version_to_buildstring }} + python: + entry_points: + - mamba = mamba.mamba:main + requirements: + build: + - if: build_platform != target_platform + then: + - python + - cross-python_${{ target_platform }} + run: + - python + - conda >=23.9,<24 + - ${{ pin_subpackage('libmambapy', exact=True) }} + + tests: + - python: + imports: + - mamba + - script: + - mamba --help + # for some reason tqdm doesn't have a proper colorama dependency so pip check fails + # but that's completely unrelated to mamba + - python -c "import mamba._version; assert mamba._version.__version__ == '${{ mamba_version }}'" + + - if: linux + then: + - test -f ${PREFIX}/etc/profile.d/mamba.sh + # these tests work when run on win, but for some reason not during conda build + - mamba create -n test_py2 python=2.7 --dry-run + - mamba install xtensor xsimd -c conda-forge --dry-run + - if: unix + then: + - test -f ${PREFIX}/condabin/mamba + +about: + homepage: https://github.com/mamba-org/mamba + license: BSD-3-Clause + license_file: LICENSE + license_family: BSD + summary: A fast drop-in alternative to conda, using libsolv for dependency resolution + description: | + # Mamba, the Fast Cross-Platform Package Manager + ${{ env.get("MY_ENV_VAR") }} + ${{ env.get("MY_ENV_VAR", default="default_value") }} + + repository: https://github.com/mamba-org/mamba diff --git a/tests/test_jinja.py b/tests/test_jinja.py index 3feb3a0..2b60dd8 100644 --- a/tests/test_jinja.py +++ b/tests/test_jinja.py @@ -5,7 +5,7 @@ def test_render_recipe_with_context(snapshot) -> None: - recipe = Path("tests/data/context.yaml") + recipe = Path("tests/data/mamba_recipe.yaml") with recipe.open() as f: recipe_yaml = load_yaml(f)