From d644905e8bdea4356efbea8f250a8a06ebc70159 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 20 Sep 2024 16:43:55 +0200 Subject: [PATCH 1/6] start implementing solvability check with rattler-build --- .../rattler_build.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 conda_forge_feedstock_check_solvable/rattler_build.py diff --git a/conda_forge_feedstock_check_solvable/rattler_build.py b/conda_forge_feedstock_check_solvable/rattler_build.py new file mode 100644 index 0000000..164f058 --- /dev/null +++ b/conda_forge_feedstock_check_solvable/rattler_build.py @@ -0,0 +1,62 @@ +import subprocess +import os +from conda_forge_feedstock_check_solvable.virtual_packages import ( + virtual_package_repodata, +) + + +def run_rattler_build(command): + try: + # Run the command and capture output + print(" ".join(command)) + result = subprocess.run(command, shell=True, check=False, capture_output=True, text=True) + + # Get the status code + status_code = result.returncode + + # Get stdout and stderr + stdout = result.stdout.strip() + stderr = result.stderr.strip() + + return status_code, stdout, stderr + except Exception as e: + return -1, "", str(e) + + +def invoke_rattler_build(recipe_dir: str, channels, build_platform, host_platform, variants) -> (bool, str): + print("invoke_rattler_build") + virtual_package_repo_url = virtual_package_repodata() + + channels_args = [] + for c in channels: + channels_args.extend(["-c", c]) + + channels_args.extend(["-c", virtual_package_repo_url]) + + variants_args = [] + # for v in variants: + # variants_args.extend(["-m", v]) + args = [ + "rattler-build", + "build", "--recipe", + recipe_dir] + channels_args + \ + ["--target-platform", host_platform, "--build-platform", build_platform] + \ + variants_args + \ + ["--render-only", "--with-solve"] + print(" ".join(args)) + + recipe = os.path.join(recipe_dir, "recipe.yaml") + status, out, err = run_rattler_build([ + "rattler-build", + "build", "--recipe", + recipe] # + channels_args + \ + # ["--target-platform", host_platform, "--build-platform", build_platform] + \ + # variants_args + \ + # ["--render-only", "--with-solve"] + ) + + if status == 0: + print(out, err) + return True, "" + else: + return False, out + err From 3b23858751ede23879cfbc42eecba2a83eec2388 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 20 Sep 2024 16:55:14 +0200 Subject: [PATCH 2/6] lint --- .../check_solvable.py | 95 ++++++++++++------- .../rattler_build.py | 33 ++++--- tests/test_check_solvable.py | 18 ++++ 3 files changed, 96 insertions(+), 50 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 38bebc0..8666c10 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -10,6 +10,7 @@ import conda_forge_feedstock_check_solvable.utils from conda_forge_feedstock_check_solvable.mamba_solver import mamba_solver_factory +from conda_forge_feedstock_check_solvable.rattler_build import invoke_rattler_build from conda_forge_feedstock_check_solvable.rattler_solver import rattler_solver_factory from conda_forge_feedstock_check_solvable.utils import ( MAX_GLIBC_MINOR, @@ -24,7 +25,6 @@ print_warning, remove_reqs_by_name, replace_pin_compatible, - suppress_output, ) from conda_forge_feedstock_check_solvable.virtual_packages import ( virtual_package_repodata, @@ -132,7 +132,12 @@ def _is_recipe_solvable( print_warning(errors[-1]) return False, errors, {} - if not os.path.exists(os.path.join(feedstock_dir, "recipe", "meta.yaml")): + meta_exists = os.path.exists(os.path.join(feedstock_dir, "recipe", "meta.yaml")) + recipe_exists = os.path.exists( + os.path.join(feedstock_dir, "recipe", "recipe.yaml") + ) + + if not (meta_exists or recipe_exists): errors.append( "No `recipe/meta.yaml` file found! This issue is quite weird and " "someone should investigate!", @@ -187,8 +192,8 @@ def _is_recipe_solvable( def _is_recipe_solvable_on_platform( - recipe_dir, - cbc_path, + recipe_dir: str, + cbc_path: str, platform, arch, build_platform_arch=None, @@ -234,43 +239,63 @@ def _is_recipe_solvable_on_platform( # it would be used in a real build print_debug("rendering recipe with conda build") - with suppress_output(): - for att in range(2): - timeout_timer.raise_for_timeout() - try: - if att == 1: - os.system("rm -f %s/conda_build_config.yaml" % recipe_dir) - config = conda_build.config.get_or_merge_config( - None, - platform=platform, - arch=arch, - variant_config_files=[cbc_path], - ) - cbc, _ = conda_build.variants.get_package_combined_spec( - recipe_dir, - config=config, - ) - except Exception as e: - if att == 0: - pass - else: - raise e - + # with suppress_output(): + for att in range(2): timeout_timer.raise_for_timeout() + try: + if att == 1: + os.system("rm -f %s/conda_build_config.yaml" % recipe_dir) + config = conda_build.config.get_or_merge_config( + None, + platform=platform, + arch=arch, + variant_config_files=[cbc_path], + ) + cbc, _ = conda_build.variants.get_package_combined_spec( + recipe_dir, + config=config, + ) + except Exception as e: + if att == 0: + pass + else: + raise e + + timeout_timer.raise_for_timeout() - # now we render the meta.yaml into an actual recipe - metas = conda_build_api_render( + # now we render the meta.yaml into an actual recipe + print("Recipe dir: ", recipe_dir) + + if os.path.exists(os.path.join(recipe_dir, "recipe.yaml")): + print("Recipe YAMLING!") + # this is a rattler-build recipe so we can invoke rattler-build with + # the new `cbc`. + solvable, errors = invoke_rattler_build( recipe_dir, - platform=platform, - arch=arch, - ignore_system_variants=True, + channels=channel_sources, + build_platform=f"{platform}-{arch}", + host_platform=f"{platform}-{arch}", variants=cbc, - permit_undefined_jinja=True, - finalize=False, - bypass_env_check=True, - channel_urls=channel_sources, ) + if errors: + print_warning("Rattler build errors: %s", errors) + errors = [f"Rattler build errors: {errors}"] + + return solvable, errors + + metas = conda_build_api_render( + recipe_dir, + platform=platform, + arch=arch, + ignore_system_variants=True, + variants=cbc, + permit_undefined_jinja=True, + finalize=False, + bypass_env_check=True, + channel_urls=channel_sources, + ) + timeout_timer.raise_for_timeout() # get build info diff --git a/conda_forge_feedstock_check_solvable/rattler_build.py b/conda_forge_feedstock_check_solvable/rattler_build.py index 164f058..cc4f02e 100644 --- a/conda_forge_feedstock_check_solvable/rattler_build.py +++ b/conda_forge_feedstock_check_solvable/rattler_build.py @@ -1,5 +1,6 @@ -import subprocess import os +import subprocess + from conda_forge_feedstock_check_solvable.virtual_packages import ( virtual_package_repodata, ) @@ -9,7 +10,9 @@ def run_rattler_build(command): try: # Run the command and capture output print(" ".join(command)) - result = subprocess.run(command, shell=True, check=False, capture_output=True, text=True) + result = subprocess.run( + command, shell=True, check=False, capture_output=True, text=True + ) # Get the status code status_code = result.returncode @@ -23,7 +26,9 @@ def run_rattler_build(command): return -1, "", str(e) -def invoke_rattler_build(recipe_dir: str, channels, build_platform, host_platform, variants) -> (bool, str): +def invoke_rattler_build( + recipe_dir: str, channels, build_platform, host_platform, variants +) -> (bool, str): print("invoke_rattler_build") virtual_package_repo_url = virtual_package_repodata() @@ -35,21 +40,19 @@ def invoke_rattler_build(recipe_dir: str, channels, build_platform, host_platfor variants_args = [] # for v in variants: - # variants_args.extend(["-m", v]) - args = [ - "rattler-build", - "build", "--recipe", - recipe_dir] + channels_args + \ - ["--target-platform", host_platform, "--build-platform", build_platform] + \ - variants_args + \ - ["--render-only", "--with-solve"] + # variants_args.extend(["-m", v]) + args = ( + ["rattler-build", "build", "--recipe", recipe_dir] + + channels_args + + ["--target-platform", host_platform, "--build-platform", build_platform] + + variants_args + + ["--render-only", "--with-solve"] + ) print(" ".join(args)) recipe = os.path.join(recipe_dir, "recipe.yaml") - status, out, err = run_rattler_build([ - "rattler-build", - "build", "--recipe", - recipe] # + channels_args + \ + status, out, err = run_rattler_build( + ["rattler-build", "build", "--recipe", recipe] # + channels_args + \ # ["--target-platform", host_platform, "--build-platform", build_platform] + \ # variants_args + \ # ["--render-only", "--with-solve"] diff --git a/tests/test_check_solvable.py b/tests/test_check_solvable.py index 3271370..aacdbb1 100644 --- a/tests/test_check_solvable.py +++ b/tests/test_check_solvable.py @@ -572,3 +572,21 @@ def test_pillow_solvable(tmp_path, solver): pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) assert any("python3.10" in k for k in solvable_by_variant) + + +def test_jolt_physics_rattler(tmp_path): + """test the new recipe format""" + feedstock_dir = clone_and_checkout_repo( + tmp_path, + "https://github.com/conda-forge/jolt-physics-feedstock", + ref="main", + ) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver="rattler", + verbosity=VERB, + timeout=None, + fail_fast=True, + ) + pprint.pprint(solvable_by_variant) + assert solvable, pprint.pformat(errors) From fd2357b1fbf12a0c3bd6588ae1ad039fc7f6d2d9 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Fri, 20 Sep 2024 17:18:56 +0200 Subject: [PATCH 3/6] make it work --- .../rattler_build.py | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/rattler_build.py b/conda_forge_feedstock_check_solvable/rattler_build.py index cc4f02e..34ef766 100644 --- a/conda_forge_feedstock_check_solvable/rattler_build.py +++ b/conda_forge_feedstock_check_solvable/rattler_build.py @@ -1,5 +1,7 @@ -import os import subprocess +import tempfile + +import yaml from conda_forge_feedstock_check_solvable.virtual_packages import ( virtual_package_repodata, @@ -9,9 +11,9 @@ def run_rattler_build(command): try: # Run the command and capture output - print(" ".join(command)) + print("Running: ", " ".join(command)) result = subprocess.run( - command, shell=True, check=False, capture_output=True, text=True + " ".join(command), shell=True, check=False, capture_output=True, text=True ) # Get the status code @@ -29,37 +31,33 @@ def run_rattler_build(command): def invoke_rattler_build( recipe_dir: str, channels, build_platform, host_platform, variants ) -> (bool, str): - print("invoke_rattler_build") + # this is OK since there is an lru cache virtual_package_repo_url = virtual_package_repodata() + # create a temporary file and dump the variants as YAML + + with tempfile.NamedTemporaryFile(mode="w", delete=False) as variants_file: + yaml.dump(variants, variants_file) + variants_file.flush() - channels_args = [] - for c in channels: - channels_args.extend(["-c", c]) + channels_args = [] + for c in channels: + channels_args.extend(["-c", c]) - channels_args.extend(["-c", virtual_package_repo_url]) + channels_args.extend(["-c", virtual_package_repo_url]) - variants_args = [] - # for v in variants: - # variants_args.extend(["-m", v]) - args = ( - ["rattler-build", "build", "--recipe", recipe_dir] - + channels_args - + ["--target-platform", host_platform, "--build-platform", build_platform] - + variants_args - + ["--render-only", "--with-solve"] - ) - print(" ".join(args)) + args = ( + ["rattler-build", "build", "--recipe", recipe_dir] + + channels_args + + ["--target-platform", host_platform] + + ["--build-platform", build_platform] + + ["-m", variants_file.name] + + ["--render-only", "--with-solve"] + ) - recipe = os.path.join(recipe_dir, "recipe.yaml") - status, out, err = run_rattler_build( - ["rattler-build", "build", "--recipe", recipe] # + channels_args + \ - # ["--target-platform", host_platform, "--build-platform", build_platform] + \ - # variants_args + \ - # ["--render-only", "--with-solve"] - ) + status, out, err = run_rattler_build(args) - if status == 0: - print(out, err) - return True, "" - else: - return False, out + err + if status == 0: + print(out, err) + return True, "" + else: + return False, out + err From 211c931c6e55b0e31d941346a279bca203e89c16 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Sat, 21 Sep 2024 10:27:10 +0200 Subject: [PATCH 4/6] restore suppress output --- .../check_solvable.py | 95 ++++++++++--------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 8666c10..e61edde 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -25,6 +25,7 @@ print_warning, remove_reqs_by_name, replace_pin_compatible, + suppress_output, ) from conda_forge_feedstock_check_solvable.virtual_packages import ( virtual_package_repodata, @@ -239,63 +240,63 @@ def _is_recipe_solvable_on_platform( # it would be used in a real build print_debug("rendering recipe with conda build") - # with suppress_output(): - for att in range(2): + with suppress_output(): + for att in range(2): + timeout_timer.raise_for_timeout() + try: + if att == 1: + os.system("rm -f %s/conda_build_config.yaml" % recipe_dir) + config = conda_build.config.get_or_merge_config( + None, + platform=platform, + arch=arch, + variant_config_files=[cbc_path], + ) + cbc, _ = conda_build.variants.get_package_combined_spec( + recipe_dir, + config=config, + ) + except Exception as e: + if att == 0: + pass + else: + raise e + timeout_timer.raise_for_timeout() - try: - if att == 1: - os.system("rm -f %s/conda_build_config.yaml" % recipe_dir) - config = conda_build.config.get_or_merge_config( - None, - platform=platform, - arch=arch, - variant_config_files=[cbc_path], - ) - cbc, _ = conda_build.variants.get_package_combined_spec( + + # now we render the meta.yaml into an actual recipe + print("Recipe dir: ", recipe_dir) + + if os.path.exists(os.path.join(recipe_dir, "recipe.yaml")): + print("Recipe YAMLING!") + # this is a rattler-build recipe so we can invoke rattler-build with + # the new `cbc`. + solvable, errors = invoke_rattler_build( recipe_dir, - config=config, + channels=channel_sources, + build_platform=f"{platform}-{arch}", + host_platform=f"{platform}-{arch}", + variants=cbc, ) - except Exception as e: - if att == 0: - pass - else: - raise e - timeout_timer.raise_for_timeout() + if errors: + print_warning("Rattler build errors: %s", errors) + errors = [f"Rattler build errors: {errors}"] - # now we render the meta.yaml into an actual recipe - print("Recipe dir: ", recipe_dir) + return solvable, errors - if os.path.exists(os.path.join(recipe_dir, "recipe.yaml")): - print("Recipe YAMLING!") - # this is a rattler-build recipe so we can invoke rattler-build with - # the new `cbc`. - solvable, errors = invoke_rattler_build( + metas = conda_build_api_render( recipe_dir, - channels=channel_sources, - build_platform=f"{platform}-{arch}", - host_platform=f"{platform}-{arch}", + platform=platform, + arch=arch, + ignore_system_variants=True, variants=cbc, + permit_undefined_jinja=True, + finalize=False, + bypass_env_check=True, + channel_urls=channel_sources, ) - if errors: - print_warning("Rattler build errors: %s", errors) - errors = [f"Rattler build errors: {errors}"] - - return solvable, errors - - metas = conda_build_api_render( - recipe_dir, - platform=platform, - arch=arch, - ignore_system_variants=True, - variants=cbc, - permit_undefined_jinja=True, - finalize=False, - bypass_env_check=True, - channel_urls=channel_sources, - ) - timeout_timer.raise_for_timeout() # get build info From a5850bac8071edb5a9dcf4186b6895a27fbd5799 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Sun, 22 Sep 2024 22:06:23 +0200 Subject: [PATCH 5/6] add a test for an unsolvable recipe --- tests/test_check_solvable.py | 15 +++++++++++++++ .../.ci_support/osx_arm64_.yaml | 4 ++++ tests/v1-unsolvable-feedstock/recipe/recipe.yaml | 7 +++++++ 3 files changed, 26 insertions(+) create mode 100644 tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml create mode 100644 tests/v1-unsolvable-feedstock/recipe/recipe.yaml diff --git a/tests/test_check_solvable.py b/tests/test_check_solvable.py index aacdbb1..1582276 100644 --- a/tests/test_check_solvable.py +++ b/tests/test_check_solvable.py @@ -590,3 +590,18 @@ def test_jolt_physics_rattler(tmp_path): ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) + + +def test_v1_unsolvable(tmp_path): + """test an unsolvable recipe in the new format""" + feedstock_dir = os.path.join(os.path.dirname(__file__), "v1-unsolvable-feedstock") + solvable, errors, _ = is_recipe_solvable( + feedstock_dir, + solver="rattler", + verbosity=VERB, + timeout=None, + fail_fast=True, + ) + assert solvable is False + + print(errors[0]) diff --git a/tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml b/tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml new file mode 100644 index 0000000..5a29dfb --- /dev/null +++ b/tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml @@ -0,0 +1,4 @@ +python: + - 100 +channel_sources: + - conda-forge \ No newline at end of file diff --git a/tests/v1-unsolvable-feedstock/recipe/recipe.yaml b/tests/v1-unsolvable-feedstock/recipe/recipe.yaml new file mode 100644 index 0000000..db83835 --- /dev/null +++ b/tests/v1-unsolvable-feedstock/recipe/recipe.yaml @@ -0,0 +1,7 @@ +package: + name: v1-unsolvable + version: "0.1" + +requirements: + build: + - python \ No newline at end of file From 0df0679ccb941ff001060f560fe854a1c2bd0773 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Sun, 22 Sep 2024 22:09:23 +0200 Subject: [PATCH 6/6] remove print statements --- conda_forge_feedstock_check_solvable/check_solvable.py | 5 +---- conda_forge_feedstock_check_solvable/rattler_build.py | 4 ++-- tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml | 2 +- tests/v1-unsolvable-feedstock/recipe/recipe.yaml | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index e61edde..511b306 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -264,11 +264,7 @@ def _is_recipe_solvable_on_platform( timeout_timer.raise_for_timeout() - # now we render the meta.yaml into an actual recipe - print("Recipe dir: ", recipe_dir) - if os.path.exists(os.path.join(recipe_dir, "recipe.yaml")): - print("Recipe YAMLING!") # this is a rattler-build recipe so we can invoke rattler-build with # the new `cbc`. solvable, errors = invoke_rattler_build( @@ -285,6 +281,7 @@ def _is_recipe_solvable_on_platform( return solvable, errors + # now we render the meta.yaml into an actual recipe metas = conda_build_api_render( recipe_dir, platform=platform, diff --git a/conda_forge_feedstock_check_solvable/rattler_build.py b/conda_forge_feedstock_check_solvable/rattler_build.py index 34ef766..358d535 100644 --- a/conda_forge_feedstock_check_solvable/rattler_build.py +++ b/conda_forge_feedstock_check_solvable/rattler_build.py @@ -3,6 +3,7 @@ import yaml +from conda_forge_feedstock_check_solvable.utils import print_debug from conda_forge_feedstock_check_solvable.virtual_packages import ( virtual_package_repodata, ) @@ -11,7 +12,7 @@ def run_rattler_build(command): try: # Run the command and capture output - print("Running: ", " ".join(command)) + print_debug("Running: ", " ".join(command)) result = subprocess.run( " ".join(command), shell=True, check=False, capture_output=True, text=True ) @@ -57,7 +58,6 @@ def invoke_rattler_build( status, out, err = run_rattler_build(args) if status == 0: - print(out, err) return True, "" else: return False, out + err diff --git a/tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml b/tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml index 5a29dfb..8f89a14 100644 --- a/tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml +++ b/tests/v1-unsolvable-feedstock/.ci_support/osx_arm64_.yaml @@ -1,4 +1,4 @@ python: - 100 channel_sources: - - conda-forge \ No newline at end of file + - conda-forge diff --git a/tests/v1-unsolvable-feedstock/recipe/recipe.yaml b/tests/v1-unsolvable-feedstock/recipe/recipe.yaml index db83835..c38a8c5 100644 --- a/tests/v1-unsolvable-feedstock/recipe/recipe.yaml +++ b/tests/v1-unsolvable-feedstock/recipe/recipe.yaml @@ -4,4 +4,4 @@ package: requirements: build: - - python \ No newline at end of file + - python