From 2ca9916b67a2fcaded6ba47ff8ba28fafc906c12 Mon Sep 17 00:00:00 2001 From: beckermr Date: Sat, 1 Jun 2024 13:52:47 -0500 Subject: [PATCH 01/35] WIP first pass at more refactors for rattler --- .../check_solvable.py | 12 +- .../mamba_solver.py | 161 +---------------- .../rattler_solver.py | 0 .../virtual_packages.py | 164 ++++++++++++++++++ tests/test_mamba_solvable.py | 10 +- 5 files changed, 177 insertions(+), 170 deletions(-) create mode 100644 conda_forge_feedstock_check_solvable/rattler_solver.py create mode 100644 conda_forge_feedstock_check_solvable/virtual_packages.py diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 455c8a8..5989e2d 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -6,10 +6,7 @@ import psutil from ruamel.yaml import YAML -from conda_forge_feedstock_check_solvable.mamba_solver import ( - _mamba_factory, - virtual_package_repodata, -) +from conda_forge_feedstock_check_solvable.mamba_solver import mamba_solver_factory from conda_forge_feedstock_check_solvable.utils import ( MAX_GLIBC_MINOR, apply_pins, @@ -20,6 +17,9 @@ remove_reqs_by_name, suppress_output, ) +from conda_forge_feedstock_check_solvable.virtual_packages import ( + virtual_package_repodata, +) def _func(feedstock_dir, additional_channels, build_platform, verbosity, conn): @@ -283,8 +283,8 @@ def _is_recipe_solvable_on_platform( # we check run and host and ignore the rest print_debug("getting mamba solver") with suppress_output(): - solver = _mamba_factory(tuple(channel_sources), f"{platform}-{arch}") - build_solver = _mamba_factory( + solver = mamba_solver_factory(tuple(channel_sources), f"{platform}-{arch}") + build_solver = mamba_solver_factory( tuple(channel_sources), f"{build_platform}-{build_arch}", ) diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index a95a6d3..93e6054 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -10,18 +10,9 @@ https://gist.github.com/wolfv/cd12bd4a448c77ff02368e97ffdf495a. """ -import atexit import copy -import functools -import os -import pathlib import pprint -import subprocess -import tempfile -import time -from collections import defaultdict -from dataclasses import dataclass, field -from typing import Dict, FrozenSet, Iterable, List, Set, Tuple +from typing import List, Tuple import cachetools.func import libmambapy as api @@ -31,12 +22,7 @@ from conda_forge_feedstock_check_solvable.mamba_utils import load_channels from conda_forge_feedstock_check_solvable.utils import ( - ALL_PLATFORMS, DEFAULT_RUN_EXPORTS, - MAX_GLIBC_MINOR, - MINIMUM_CUDA_VERS, - MINIMUM_OSX_64_VERS, - MINIMUM_OSX_ARM64_VERS, convert_spec_to_conda_build, get_run_exports, print_debug, @@ -54,81 +40,6 @@ api.Context().channel_priority = api.ChannelPriority.kStrict -@dataclass(frozen=True) -class FakePackage: - name: str - version: str = "1.0" - build_string: str = "" - build_number: int = 0 - noarch: str = "" - depends: FrozenSet[str] = field(default_factory=frozenset) - timestamp: int = field( - default_factory=lambda: int(time.mktime(time.gmtime()) * 1000), - ) - - def to_repodata_entry(self): - out = self.__dict__.copy() - if self.build_string: - build = f"{self.build_string}_{self.build_number}" - else: - build = f"{self.build_number}" - out["depends"] = list(out["depends"]) - out["build"] = build - fname = f"{self.name}-{self.version}-{build}.tar.bz2" - return fname, out - - -class FakeRepoData: - def __init__(self, base_dir: pathlib.Path): - self.base_path = base_dir - self.packages_by_subdir: Dict[FakePackage, Set[str]] = defaultdict(set) - - @property - def channel_url(self): - return f"file://{str(self.base_path.absolute())}" - - def add_package(self, package: FakePackage, subdirs: Iterable[str] = ()): - subdirs = frozenset(subdirs) - if not subdirs: - subdirs = frozenset(["noarch"]) - self.packages_by_subdir[package].update(subdirs) - - def _write_subdir(self, subdir): - packages = {} - out = {"info": {"subdir": subdir}, "packages": packages} - for pkg, subdirs in self.packages_by_subdir.items(): - if subdir not in subdirs: - continue - fname, info_dict = pkg.to_repodata_entry() - info_dict["subdir"] = subdir - packages[fname] = info_dict - - (self.base_path / subdir).mkdir(exist_ok=True) - (self.base_path / subdir / "repodata.json").write_text(json.dumps(out)) - - def write(self): - all_subdirs = ALL_PLATFORMS.copy() - all_subdirs.add("noarch") - for subdirs in self.packages_by_subdir.values(): - all_subdirs.update(subdirs) - - for subdir in all_subdirs: - self._write_subdir(subdir) - - print_debug("Wrote fake repodata to %s", self.base_path) - import glob - - for filename in glob.iglob(str(self.base_path / "**"), recursive=True): - print_debug(filename) - print_debug("repo: %s", self.channel_url) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.write() - - class MambaSolver: """Run the mamba solver. @@ -296,73 +207,5 @@ def _get_run_exports( @cachetools.func.ttl_cache(maxsize=8, ttl=60) -def _mamba_factory(channels, platform): +def mamba_solver_factory(channels, platform): return MambaSolver(list(channels), platform) - - -@functools.lru_cache(maxsize=1) -def virtual_package_repodata(): - # TODO: we might not want to use TemporaryDirectory - import shutil - - # tmp directory in github actions - runner_tmp = os.environ.get("RUNNER_TEMP") - tmp_dir = tempfile.mkdtemp(dir=runner_tmp) - - if not runner_tmp: - # no need to bother cleaning up on CI - def clean(): - shutil.rmtree(tmp_dir, ignore_errors=True) - - atexit.register(clean) - - tmp_path = pathlib.Path(tmp_dir) - repodata = FakeRepoData(tmp_path) - - # glibc - for glibc_minor in range(12, MAX_GLIBC_MINOR + 1): - repodata.add_package(FakePackage("__glibc", "2.%d" % glibc_minor)) - - # cuda - get from cuda-version on conda-forge - try: - cuda_pkgs = json.loads( - subprocess.check_output( - "CONDA_SUBDIR=linux-64 conda search cuda-version -c conda-forge --json", - shell=True, - text=True, - stderr=subprocess.PIPE, - ) - ) - cuda_vers = [pkg["version"] for pkg in cuda_pkgs["cuda-version"]] - except Exception: - cuda_vers = [] - # extra hard coded list to make sure we don't miss anything - cuda_vers += MINIMUM_CUDA_VERS - cuda_vers = set(cuda_vers) - for cuda_ver in cuda_vers: - repodata.add_package(FakePackage("__cuda", cuda_ver)) - - for osx_ver in MINIMUM_OSX_64_VERS: - repodata.add_package(FakePackage("__osx", osx_ver), subdirs=["osx-64"]) - for osx_ver in MINIMUM_OSX_ARM64_VERS: - repodata.add_package( - FakePackage("__osx", osx_ver), subdirs=["osx-arm64", "osx-64"] - ) - - repodata.add_package( - FakePackage("__win", "0"), - subdirs=list(subdir for subdir in ALL_PLATFORMS if subdir.startswith("win")), - ) - repodata.add_package( - FakePackage("__linux", "0"), - subdirs=list(subdir for subdir in ALL_PLATFORMS if subdir.startswith("linux")), - ) - repodata.add_package( - FakePackage("__unix", "0"), - subdirs=list( - subdir for subdir in ALL_PLATFORMS if not subdir.startswith("win") - ), - ) - repodata.write() - - return repodata.channel_url diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py new file mode 100644 index 0000000..e69de29 diff --git a/conda_forge_feedstock_check_solvable/virtual_packages.py b/conda_forge_feedstock_check_solvable/virtual_packages.py new file mode 100644 index 0000000..948440c --- /dev/null +++ b/conda_forge_feedstock_check_solvable/virtual_packages.py @@ -0,0 +1,164 @@ +import atexit +import functools +import os +import pathlib +import subprocess +import tempfile +import time +from collections import defaultdict +from dataclasses import dataclass, field +from typing import Dict, FrozenSet, Iterable, Set + +import rapidjson as json + +from conda_forge_feedstock_check_solvable.utils import ( + ALL_PLATFORMS, + MAX_GLIBC_MINOR, + MINIMUM_CUDA_VERS, + MINIMUM_OSX_64_VERS, + MINIMUM_OSX_ARM64_VERS, + print_debug, +) + + +@dataclass(frozen=True) +class FakePackage: + name: str + version: str = "1.0" + build_string: str = "" + build_number: int = 0 + noarch: str = "" + depends: FrozenSet[str] = field(default_factory=frozenset) + timestamp: int = field( + default_factory=lambda: int(time.mktime(time.gmtime()) * 1000), + ) + + def to_repodata_entry(self): + out = self.__dict__.copy() + if self.build_string: + build = f"{self.build_string}_{self.build_number}" + else: + build = f"{self.build_number}" + out["depends"] = list(out["depends"]) + out["build"] = build + fname = f"{self.name}-{self.version}-{build}.tar.bz2" + return fname, out + + +class FakeRepoData: + def __init__(self, base_dir: pathlib.Path): + self.base_path = base_dir + self.packages_by_subdir: Dict[FakePackage, Set[str]] = defaultdict(set) + + @property + def channel_url(self): + return f"file://{str(self.base_path.absolute())}" + + def add_package(self, package: FakePackage, subdirs: Iterable[str] = ()): + subdirs = frozenset(subdirs) + if not subdirs: + subdirs = frozenset(["noarch"]) + self.packages_by_subdir[package].update(subdirs) + + def _write_subdir(self, subdir): + packages = {} + out = {"info": {"subdir": subdir}, "packages": packages} + for pkg, subdirs in self.packages_by_subdir.items(): + if subdir not in subdirs: + continue + fname, info_dict = pkg.to_repodata_entry() + info_dict["subdir"] = subdir + packages[fname] = info_dict + + (self.base_path / subdir).mkdir(exist_ok=True) + (self.base_path / subdir / "repodata.json").write_text(json.dumps(out)) + + def write(self): + all_subdirs = ALL_PLATFORMS.copy() + all_subdirs.add("noarch") + for subdirs in self.packages_by_subdir.values(): + all_subdirs.update(subdirs) + + for subdir in all_subdirs: + self._write_subdir(subdir) + + print_debug("Wrote fake repodata to %s", self.base_path) + import glob + + for filename in glob.iglob(str(self.base_path / "**"), recursive=True): + print_debug(filename) + print_debug("repo: %s", self.channel_url) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.write() + + +@functools.lru_cache(maxsize=1) +def virtual_package_repodata(): + # TODO: we might not want to use TemporaryDirectory + import shutil + + # tmp directory in github actions + runner_tmp = os.environ.get("RUNNER_TEMP") + tmp_dir = tempfile.mkdtemp(dir=runner_tmp) + + if not runner_tmp: + # no need to bother cleaning up on CI + def clean(): + shutil.rmtree(tmp_dir, ignore_errors=True) + + atexit.register(clean) + + tmp_path = pathlib.Path(tmp_dir) + repodata = FakeRepoData(tmp_path) + + # glibc + for glibc_minor in range(12, MAX_GLIBC_MINOR + 1): + repodata.add_package(FakePackage("__glibc", "2.%d" % glibc_minor)) + + # cuda - get from cuda-version on conda-forge + try: + cuda_pkgs = json.loads( + subprocess.check_output( + "CONDA_SUBDIR=linux-64 conda search cuda-version -c conda-forge --json", + shell=True, + text=True, + stderr=subprocess.PIPE, + ) + ) + cuda_vers = [pkg["version"] for pkg in cuda_pkgs["cuda-version"]] + except Exception: + cuda_vers = [] + # extra hard coded list to make sure we don't miss anything + cuda_vers += MINIMUM_CUDA_VERS + cuda_vers = set(cuda_vers) + for cuda_ver in cuda_vers: + repodata.add_package(FakePackage("__cuda", cuda_ver)) + + for osx_ver in MINIMUM_OSX_64_VERS: + repodata.add_package(FakePackage("__osx", osx_ver), subdirs=["osx-64"]) + for osx_ver in MINIMUM_OSX_ARM64_VERS: + repodata.add_package( + FakePackage("__osx", osx_ver), subdirs=["osx-arm64", "osx-64"] + ) + + repodata.add_package( + FakePackage("__win", "0"), + subdirs=list(subdir for subdir in ALL_PLATFORMS if subdir.startswith("win")), + ) + repodata.add_package( + FakePackage("__linux", "0"), + subdirs=list(subdir for subdir in ALL_PLATFORMS if subdir.startswith("linux")), + ) + repodata.add_package( + FakePackage("__unix", "0"), + subdirs=list( + subdir for subdir in ALL_PLATFORMS if not subdir.startswith("win") + ), + ) + repodata.write() + + return repodata.channel_url diff --git a/tests/test_mamba_solvable.py b/tests/test_mamba_solvable.py index 745ab1f..f55cd79 100644 --- a/tests/test_mamba_solvable.py +++ b/tests/test_mamba_solvable.py @@ -13,7 +13,7 @@ FakePackage, FakeRepoData, MambaSolver, - _mamba_factory, + mamba_solver_factory, virtual_package_repodata, ) from conda_forge_feedstock_check_solvable.utils import apply_pins, suppress_output @@ -90,7 +90,7 @@ def test_mamba_solver_apply_pins(tmp_path): config=config, ) - solver = _mamba_factory(("conda-forge", "defaults"), "linux-64") + solver = mamba_solver_factory(("conda-forge", "defaults"), "linux-64") metas = conda_build.api.render( str(tmp_path), @@ -127,7 +127,7 @@ def test_mamba_solver_apply_pins(tmp_path): @flaky def test_mamba_solver_constraints(): with suppress_output(): - solver = _mamba_factory(("conda-forge",), "osx-64") + solver = mamba_solver_factory(("conda-forge",), "osx-64") solvable, err, solution = solver.solve( ["simplejson"], constraints=["python=3.10", "zeromq=4.2"] ) @@ -565,7 +565,7 @@ def test_virtual_package(feedstock_dir, tmp_path_factory): @flaky def test_mamba_solver_hangs(): with suppress_output(): - solver = _mamba_factory(("conda-forge", "defaults"), "osx-64") + solver = mamba_solver_factory(("conda-forge", "defaults"), "osx-64") res = solver.solve( [ "pytest", @@ -601,7 +601,7 @@ def test_mamba_solver_hangs(): assert res[0] with suppress_output(): - solver = _mamba_factory(("conda-forge", "defaults"), "linux-64") + solver = mamba_solver_factory(("conda-forge", "defaults"), "linux-64") res = solver.solve( [ "gdal >=2.1.0", From 407d737e6910776ecdba7eb598294370cb3b50dc Mon Sep 17 00:00:00 2001 From: beckermr Date: Sat, 1 Jun 2024 21:03:20 -0500 Subject: [PATCH 02/35] REF move virtual package tests --- tests/test_mamba_solvable.py | 52 ++++---------------------------- tests/test_virtual_packages.py | 54 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 47 deletions(-) create mode 100644 tests/test_virtual_packages.py diff --git a/tests/test_mamba_solvable.py b/tests/test_mamba_solvable.py index f55cd79..b262882 100644 --- a/tests/test_mamba_solvable.py +++ b/tests/test_mamba_solvable.py @@ -10,13 +10,15 @@ from conda_forge_feedstock_check_solvable.check_solvable import is_recipe_solvable from conda_forge_feedstock_check_solvable.mamba_solver import ( - FakePackage, - FakeRepoData, MambaSolver, mamba_solver_factory, - virtual_package_repodata, ) from conda_forge_feedstock_check_solvable.utils import apply_pins, suppress_output +from conda_forge_feedstock_check_solvable.virtual_packages import ( + FakePackage, + FakeRepoData, + virtual_package_repodata, +) FEEDSTOCK_DIR = os.path.join(os.path.dirname(__file__), "test_feedstock") @@ -518,50 +520,6 @@ def test_is_recipe_solvable_notok(feedstock_dir): assert not is_recipe_solvable(feedstock_dir)[0] -@flaky -def test_virtual_package(feedstock_dir, tmp_path_factory): - recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") - os.makedirs(os.path.dirname(recipe_file), exist_ok=True) - - with FakeRepoData(tmp_path_factory.mktemp("channel")) as repodata: - for pkg in [ - FakePackage("fakehostvirtualpkgdep", depends=frozenset(["__virtual >=10"])), - FakePackage("__virtual", version="10"), - ]: - repodata.add_package(pkg) - - with open(recipe_file, "w") as fp: - fp.write( - dedent( - """ - package: - name: "cf-autotick-bot-test-package" - version: "0.9" - - source: - path: . - - build: - number: 8 - - requirements: - host: - - python - - fakehostvirtualpkgdep - - pip - run: - - python - """, - ), - ) - - solvable, err, solve_by_variant = is_recipe_solvable( - feedstock_dir, - additional_channels=[repodata.channel_url], - ) - assert solvable - - @flaky def test_mamba_solver_hangs(): with suppress_output(): diff --git a/tests/test_virtual_packages.py b/tests/test_virtual_packages.py new file mode 100644 index 0000000..099c753 --- /dev/null +++ b/tests/test_virtual_packages.py @@ -0,0 +1,54 @@ +import os +from textwrap import dedent + +from flaky import flaky + +from conda_forge_feedstock_check_solvable.check_solvable import is_recipe_solvable +from conda_forge_feedstock_check_solvable.virtual_packages import ( + FakePackage, + FakeRepoData, +) + + +@flaky +def test_virtual_package(feedstock_dir, tmp_path_factory): + recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") + os.makedirs(os.path.dirname(recipe_file), exist_ok=True) + + with FakeRepoData(tmp_path_factory.mktemp("channel")) as repodata: + for pkg in [ + FakePackage("fakehostvirtualpkgdep", depends=frozenset(["__virtual >=10"])), + FakePackage("__virtual", version="10"), + ]: + repodata.add_package(pkg) + + with open(recipe_file, "w") as fp: + fp.write( + dedent( + """ + package: + name: "cf-autotick-bot-test-package" + version: "0.9" + + source: + path: . + + build: + number: 8 + + requirements: + host: + - python + - fakehostvirtualpkgdep + - pip + run: + - python + """, + ), + ) + + solvable, err, solve_by_variant = is_recipe_solvable( + feedstock_dir, + additional_channels=[repodata.channel_url], + ) + assert solvable From af9c124751378947fa166061d2f5c84bbfff1928 Mon Sep 17 00:00:00 2001 From: beckermr Date: Sat, 1 Jun 2024 23:24:30 -0500 Subject: [PATCH 03/35] REF refactor tests to use rattler, add rattler solver --- .../check_solvable.py | 32 +- .../rattler_solver.py | 127 +++++++ .../virtual_packages.py | 13 +- ...mba_solvable.py => test_check_solvable.py} | 318 +++++------------- tests/test_solvers.py | 225 +++++++++++++ 5 files changed, 467 insertions(+), 248 deletions(-) rename tests/{test_mamba_solvable.py => test_check_solvable.py} (58%) create mode 100644 tests/test_solvers.py diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 5989e2d..2a0e1a3 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -6,7 +6,9 @@ import psutil from ruamel.yaml import YAML +import conda_forge_feedstock_check_solvable.utils from conda_forge_feedstock_check_solvable.mamba_solver import mamba_solver_factory +from conda_forge_feedstock_check_solvable.rattler_solver import rattler_solver_factory from conda_forge_feedstock_check_solvable.utils import ( MAX_GLIBC_MINOR, apply_pins, @@ -22,13 +24,14 @@ ) -def _func(feedstock_dir, additional_channels, build_platform, verbosity, conn): +def _func(feedstock_dir, additional_channels, build_platform, verbosity, solver, conn): try: res = _is_recipe_solvable( feedstock_dir, additional_channels=additional_channels, build_platform=build_platform, verbosity=verbosity, + solver=solver, ) conn.send(res) except Exception as e: @@ -43,6 +46,7 @@ def is_recipe_solvable( timeout=600, build_platform=None, verbosity=1, + solver="mamba", ) -> Tuple[bool, List[str], Dict[str, bool]]: """Compute if a recipe is solvable. @@ -64,6 +68,8 @@ def is_recipe_solvable( verbosity : int An int indicating the level of verbosity from 0 (no output) to 3 (gobbs of output). + solver : str + The solver to use. One of `mamba` or `rattler`. Returns ------- @@ -71,7 +77,7 @@ def is_recipe_solvable( The logical AND of the solvability of the recipe on all platforms in the CI scripts. errors : list of str - A list of errors from mamba. Empty if recipe is solvable. + A list of errors from the solver. Empty if recipe is solvable. solvable_by_variant : dict A lookup by variant config that shows if a particular config is solvable """ @@ -86,6 +92,7 @@ def is_recipe_solvable( additional_channels, build_platform, verbosity, + solver, child_conn, ), ) @@ -121,6 +128,7 @@ def is_recipe_solvable( additional_channels=additional_channels, build_platform=build_platform, verbosity=verbosity, + solver=solver, ) return res @@ -131,9 +139,9 @@ def _is_recipe_solvable( additional_channels=(), build_platform=None, verbosity=1, + solver="mamba", ) -> Tuple[bool, List[str], Dict[str, bool]]: - global VERBOSITY - VERBOSITY = verbosity + conda_forge_feedstock_check_solvable.utils.VERBOSITY = verbosity build_platform = build_platform or {} @@ -187,6 +195,7 @@ def _is_recipe_solvable( build_platform.get(f"{platform}_{arch}", f"{platform}_{arch}") ), additional_channels=additional_channels, + solver_backend=solver, ) solvable = solvable and _solvable cbc_name = os.path.basename(cbc_fname).rsplit(".", maxsplit=1)[0] @@ -205,6 +214,7 @@ def _is_recipe_solvable_on_platform( arch, build_platform_arch=None, additional_channels=(), + solver_backend="mamba", ): # parse the channel sources from the CBC parser = YAML(typ="jinja2") @@ -281,13 +291,21 @@ def _is_recipe_solvable_on_platform( # now we loop through each one and check if we can solve it # we check run and host and ignore the rest - print_debug("getting mamba solver") + print_debug("getting solver") with suppress_output(): - solver = mamba_solver_factory(tuple(channel_sources), f"{platform}-{arch}") - build_solver = mamba_solver_factory( + if solver_backend == "rattler": + solver_factory = rattler_solver_factory + elif solver_backend == "mamba": + solver_factory = mamba_solver_factory + else: + raise ValueError(f"Unknown solver backend {solver_backend}") + + solver = solver_factory(tuple(channel_sources), f"{platform}-{arch}") + build_solver = solver_factory( tuple(channel_sources), f"{build_platform}-{build_arch}", ) + solvable = True errors = [] outnames = [m.name() for m, _, _ in metas] diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index e69de29..597c44c 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -0,0 +1,127 @@ +import asyncio +import copy +import os +import pprint +from typing import List + +import cachetools.func +from rattler import Channel, MatchSpec, Platform, RepoDataRecord, solve + +from conda_forge_feedstock_check_solvable.utils import ( + DEFAULT_RUN_EXPORTS, + get_run_exports, + print_debug, + print_warning, +) + + +class RattlerSolver: + def __init__(self, channels, platform_arch) -> None: + _channels = [] + for c in channels: + if c == "defaults": + _channels.append("https://repo.anaconda.com/pkgs/main") + _channels.append("https://repo.anaconda.com/pkgs/r") + _channels.append("https://repo.anaconda.com/pkgs/msys2") + else: + _channels.append(c) + self.channels = [Channel(c) for c in _channels] + self.platform_arch = platform_arch + self.platforms = [Platform(self.platform_arch), Platform("noarch")] + + def solve( + self, + specs: List[str], + get_run_exports: bool = False, + ignore_run_exports_from: List[str] = None, + ignore_run_exports: List[str] = None, + constraints=None, + ): + ignore_run_exports_from = ignore_run_exports_from or [] + ignore_run_exports = ignore_run_exports or [] + success = False + err = None + run_exports = copy.deepcopy(DEFAULT_RUN_EXPORTS) + + try: + _specs = [MatchSpec(s) for s in specs] + + print_debug( + "RATTLER running solver for specs \n\n%s\n", pprint.pformat(_specs) + ) + + solution = asyncio.run( + solve( + channels=self.channels, + specs=_specs, + platforms=self.platforms, + # virtual_packages=self.virtual_packages, + ) + ) + success = True + str_solution = [ + f"{record.name.normalized} {record.version} {record.build}" + for record in solution + ] + + if get_run_exports: + run_exports = self._get_run_exports( + solution, + _specs, + [MatchSpec(igrf) for igrf in ignore_run_exports_from], + [MatchSpec(igr) for igr in ignore_run_exports], + ) + + except Exception as e: + err = str(e) + print_warning( + "RATTLER failed to solve specs \n\n%s\n\nfor channels " + "\n\n%s\n\nThe reported errors are:\n\n%s\n", + pprint.pformat(_specs), + pprint.pformat(self.channels), + err, + ) + success = False + run_exports = copy.deepcopy(DEFAULT_RUN_EXPORTS) + str_solution = None + + if get_run_exports: + return success, err, str_solution, run_exports + else: + return success, err, str_solution + + def _get_run_exports( + self, + repodata_records: List[RepoDataRecord], + _specs: List[MatchSpec], + ignore_run_exports_from: List[MatchSpec], + ignore_run_exports: List[MatchSpec], + ): + """Given a set of repodata records, produce a + dict with the weak and strong run exports for the packages. + + We only look up export data for things explicitly listed in the original + specs. + """ + names = {s.name for s in _specs} + ign_rex_from = {s.name for s in ignore_run_exports_from} + ign_rex = {s.name for s in ignore_run_exports} + run_exports = copy.deepcopy(DEFAULT_RUN_EXPORTS) + for record in repodata_records: + lt_name = record.name + if lt_name in names and lt_name not in ign_rex_from: + channel_url = record.channel + subdir = record.subdir + file_name = record.file_name + rx = get_run_exports(os.path.join(channel_url, subdir), file_name) + for key in rx: + rx[key] = {v for v in rx[key] if v not in ign_rex} + for key in DEFAULT_RUN_EXPORTS: + run_exports[key] |= rx[key] + + return run_exports + + +@cachetools.func.ttl_cache(maxsize=8, ttl=60) +def rattler_solver_factory(channels, platform): + return RattlerSolver(list(channels), platform) diff --git a/conda_forge_feedstock_check_solvable/virtual_packages.py b/conda_forge_feedstock_check_solvable/virtual_packages.py index 948440c..3b231d7 100644 --- a/conda_forge_feedstock_check_solvable/virtual_packages.py +++ b/conda_forge_feedstock_check_solvable/virtual_packages.py @@ -62,12 +62,23 @@ def add_package(self, package: FakePackage, subdirs: Iterable[str] = ()): def _write_subdir(self, subdir): packages = {} - out = {"info": {"subdir": subdir}, "packages": packages} + out = { + "info": {"subdir": subdir}, + "packages": packages, + "paxkages.conda": {}, + "removed": [], + "repodata_version": 1, + } for pkg, subdirs in self.packages_by_subdir.items(): if subdir not in subdirs: continue fname, info_dict = pkg.to_repodata_entry() info_dict["subdir"] = subdir + if subdir == "noarch": + info_dict["noarch"] = "generic" + else: + if "noarch" in info_dict: + del info_dict["noarch"] packages[fname] = info_dict (self.base_path / subdir).mkdir(exist_ok=True) diff --git a/tests/test_mamba_solvable.py b/tests/test_check_solvable.py similarity index 58% rename from tests/test_mamba_solvable.py rename to tests/test_check_solvable.py index b262882..7e637b6 100644 --- a/tests/test_mamba_solvable.py +++ b/tests/test_check_solvable.py @@ -9,158 +9,13 @@ from flaky import flaky from conda_forge_feedstock_check_solvable.check_solvable import is_recipe_solvable -from conda_forge_feedstock_check_solvable.mamba_solver import ( - MambaSolver, - mamba_solver_factory, -) -from conda_forge_feedstock_check_solvable.utils import apply_pins, suppress_output from conda_forge_feedstock_check_solvable.virtual_packages import ( FakePackage, FakeRepoData, - virtual_package_repodata, ) FEEDSTOCK_DIR = os.path.join(os.path.dirname(__file__), "test_feedstock") - - -@flaky -def test_mamba_solver_apply_pins(tmp_path): - with open(tmp_path / "meta.yaml", "w") as fp: - fp.write( - """\ -{% set name = "cf-autotick-bot-test-package" %} -{% set version = "0.9" %} - -package: - name: {{ name|lower }} - version: {{ version }} - -source: - path: . - -build: - number: 8 - -requirements: - host: - - python - - pip - - jpeg - run: - - python - -test: - commands: - - echo "works!" - -about: - home: https://github.com/regro/cf-scripts - license: BSD-3-Clause - license_family: BSD - license_file: LICENSE - summary: testing feedstock for the regro-cf-autotick-bot - -extra: - recipe-maintainers: - - beckermr - - conda-forge/bot -""", - ) - - with open(tmp_path / "conda_build_config.yaml", "w") as fp: - fp.write( - """\ -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x -python: -- 3.8.* *_cpython -""", - ) - import conda_build.api - - with suppress_output(): - config = conda_build.config.get_or_merge_config( - None, - platform="linux", - arch="64", - variant_config_files=[], - ) - cbc, _ = conda_build.variants.get_package_combined_spec( - str(tmp_path), - config=config, - ) - - solver = mamba_solver_factory(("conda-forge", "defaults"), "linux-64") - - metas = conda_build.api.render( - str(tmp_path), - platform="linux", - arch="64", - ignore_system_variants=True, - variants=cbc, - permit_undefined_jinja=True, - finalize=False, - bypass_env_check=True, - channel_urls=("conda-forge", "defaults"), - ) - - m = metas[0][0] - outnames = [m.name() for m, _, _ in metas] - build_req = m.get_value("requirements/build", []) - host_req = m.get_value("requirements/host", []) - run_req = m.get_value("requirements/run", []) - _, _, build_req, rx = solver.solve(build_req, get_run_exports=True) - print("build req: %s" % pprint.pformat(build_req)) - print("build rex: %s" % pprint.pformat(rx)) - host_req = list(set(host_req) | rx["strong"]) - run_req = list(set(run_req) | rx["strong"]) - _, _, host_req, rx = solver.solve(host_req, get_run_exports=True) - print("host req: %s" % pprint.pformat(host_req)) - print("host rex: %s" % pprint.pformat(rx)) - run_req = list(set(run_req) | rx["weak"]) - run_req = apply_pins(run_req, host_req, build_req, outnames, m) - print("run req: %s" % pprint.pformat(run_req)) - assert any(r.startswith("python >=3.8") for r in run_req) - assert any(r.startswith("jpeg >=") for r in run_req) - - -@flaky -def test_mamba_solver_constraints(): - with suppress_output(): - solver = mamba_solver_factory(("conda-forge",), "osx-64") - solvable, err, solution = solver.solve( - ["simplejson"], constraints=["python=3.10", "zeromq=4.2"] - ) - assert solvable, err - python = [pkg for pkg in solution if pkg.split()[0] == "python"][0] - name, version, build = python.split(None, 2) - assert version.startswith("3.10.") - assert not any(pkg.startswith("zeromq") for pkg in solution), pprint.pformat( - solution - ) - - -@flaky -def test_mamba_solver_constraints_unsolvable(): - with suppress_output(): - solver = MambaSolver(("conda-forge",), "osx-64") - solvable, err, solution = solver.solve( - ["simplejson"], constraints=["python=3.10", "python=3.11"] - ) - assert not solvable, pprint.pformat(solution) - - -@flaky -def test_mamba_solver_nvcc(): - with suppress_output(): - virtual_packages = virtual_package_repodata() - solver = MambaSolver([virtual_packages, "conda-forge", "defaults"], "linux-64") - out = solver.solve( - ["gcc_linux-64 7.*", "gxx_linux-64 7.*", "nvcc_linux-64 11.0.*"] - ) - assert out[0], out[1] +VERB = 1 @pytest.fixture() @@ -173,8 +28,13 @@ def feedstock_dir(tmp_path): return str(tmp_path) +@pytest.fixture(scope="session", params=["rattler", "mamba"]) +def solver(request): + yield request.param + + @flaky -def test_is_recipe_solvable_ok(feedstock_dir): +def test_is_recipe_solvable_ok(feedstock_dir, solver): recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") os.makedirs(os.path.dirname(recipe_file), exist_ok=True) with open(recipe_file, "w") as fp: @@ -217,11 +77,11 @@ def test_is_recipe_solvable_ok(feedstock_dir): - conda-forge/bot """, ) - assert is_recipe_solvable(feedstock_dir)[0] + assert is_recipe_solvable(feedstock_dir, solver=solver, verbosity=VERB)[0] @flaky -def test_unsolvable_for_particular_python(feedstock_dir): +def test_unsolvable_for_particular_python(feedstock_dir, solver): recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") os.makedirs(os.path.dirname(recipe_file), exist_ok=True) with open(recipe_file, "w") as fp: @@ -265,7 +125,11 @@ def test_unsolvable_for_particular_python(feedstock_dir): - conda-forge/bot """, ) - solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver=solver, + verbosity=VERB, + ) print(solvable_by_variant) assert not solvable # we don't have galsim for this variant so this is an expected failure @@ -276,29 +140,37 @@ def test_unsolvable_for_particular_python(feedstock_dir): @flaky -def test_r_base_cross_solvable(): +def test_r_base_cross_solvable(solver): feedstock_dir = os.path.join(os.path.dirname(__file__), "r-base-feedstock") - solvable, errors, _ = is_recipe_solvable(feedstock_dir) + solvable, errors, _ = is_recipe_solvable( + feedstock_dir, solver=solver, verbosity=VERB + ) assert solvable, pprint.pformat(errors) solvable, errors, _ = is_recipe_solvable( feedstock_dir, build_platform={"osx_arm64": "osx_64"}, + solver=solver, + verbosity=VERB, ) assert solvable, pprint.pformat(errors) @flaky -def test_xgboost_solvable(): +def test_xgboost_solvable(solver): feedstock_dir = os.path.join(os.path.dirname(__file__), "xgboost-feedstock") - solvable, errors, _ = is_recipe_solvable(feedstock_dir) + solvable, errors, _ = is_recipe_solvable( + feedstock_dir, solver=solver, verbosity=VERB + ) assert solvable, pprint.pformat(errors) @flaky -def test_pandas_solvable(): +def test_pandas_solvable(solver): feedstock_dir = os.path.join(os.path.dirname(__file__), "pandas-feedstock") - solvable, errors, _ = is_recipe_solvable(feedstock_dir) + solvable, errors, _ = is_recipe_solvable( + feedstock_dir, solver=solver, verbosity=VERB + ) assert solvable, pprint.pformat(errors) @@ -311,58 +183,74 @@ def clone_and_checkout_repo(base_path: pathlib.Path, origin_url: str, ref: str): @flaky -def test_arrow_solvable(tmp_path): +def test_arrow_solvable(tmp_path, solver): feedstock_dir = clone_and_checkout_repo( tmp_path, "https://github.com/conda-forge/arrow-cpp-feedstock", ref="main", ) - solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver=solver, + verbosity=VERB, + ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @flaky -def test_guiqwt_solvable(tmp_path): +def test_guiqwt_solvable(tmp_path, solver): """test for run exports as a single string in pyqt""" feedstock_dir = clone_and_checkout_repo( tmp_path, "https://github.com/conda-forge/guiqwt-feedstock", ref="main", ) - solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver=solver, + verbosity=VERB, + ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @flaky -def test_datalad_solvable(tmp_path): +def test_datalad_solvable(tmp_path, solver): """has an odd thing where it hangs""" feedstock_dir = clone_and_checkout_repo( tmp_path, "https://github.com/conda-forge/datalad-feedstock", ref="main", ) - solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver=solver, + verbosity=VERB, + ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @flaky -def test_grpcio_solvable(tmp_path): +def test_grpcio_solvable(tmp_path, solver): """grpcio has a runtime dep on openssl which has strange pinning things in it""" feedstock_dir = clone_and_checkout_repo( tmp_path, "https://github.com/conda-forge/grpcio-feedstock", ref="main", ) - solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver=solver, + verbosity=VERB, + ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @flaky -def test_cupy_solvable(tmp_path): +def test_cupy_solvable(tmp_path, solver): """grpcio has a runtime dep on openssl which has strange pinning things in it""" feedstock_dir = clone_and_checkout_repo( tmp_path, @@ -374,13 +262,17 @@ def test_cupy_solvable(tmp_path): shell=True, check=True, ) - solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver=solver, + verbosity=VERB, + ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @flaky -def test_run_exports_constrains_conflict(feedstock_dir, tmp_path_factory): +def test_run_exports_constrains_conflict(feedstock_dir, tmp_path_factory, solver): recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") os.makedirs(os.path.dirname(recipe_file), exist_ok=True) @@ -428,12 +320,14 @@ def test_run_exports_constrains_conflict(feedstock_dir, tmp_path_factory): feedstock_dir, additional_channels=[repodata.channel_url], timeout=None, + solver=solver, + verbosity=VERB, ) assert solvable, pprint.pformat(errors) @flaky -def test_run_exports_constrains_notok(feedstock_dir, tmp_path_factory): +def test_run_exports_constrains_notok(feedstock_dir, tmp_path_factory, solver): recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") os.makedirs(os.path.dirname(recipe_file), exist_ok=True) @@ -468,12 +362,16 @@ def test_run_exports_constrains_notok(feedstock_dir, tmp_path_factory): for cbc in pathlib.Path(feedstock_dir).glob(".ci_support/*.yaml"): if cbc.name != "linux_python3.8.____cpython.yaml": cbc.unlink() - solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver=solver, + verbosity=VERB, + ) assert not solvable, pprint.pformat(errors) @flaky -def test_is_recipe_solvable_notok(feedstock_dir): +def test_is_recipe_solvable_notok(feedstock_dir, solver): recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") os.makedirs(os.path.dirname(recipe_file), exist_ok=True) with open(recipe_file, "w") as fp: @@ -517,77 +415,11 @@ def test_is_recipe_solvable_notok(feedstock_dir): - conda-forge/bot """, ) - assert not is_recipe_solvable(feedstock_dir)[0] - - -@flaky -def test_mamba_solver_hangs(): - with suppress_output(): - solver = mamba_solver_factory(("conda-forge", "defaults"), "osx-64") - res = solver.solve( - [ - "pytest", - "selenium", - "requests-mock", - "ncurses >=6.2,<7.0a0", - "libffi >=3.2.1,<4.0a0", - "xz >=5.2.5,<6.0a0", - "nbconvert >=5.6", - "sqlalchemy", - "jsonschema", - "six >=1.11", - "python_abi 3.9.* *_cp39", - "tornado", - "jupyter", - "requests", - "jupyter_client", - "notebook >=4.2", - "tk >=8.6.10,<8.7.0a0", - "openssl >=1.1.1h,<1.1.2a", - "readline >=8.0,<9.0a0", - "fuzzywuzzy", - "python >=3.9,<3.10.0a0", - "traitlets", - "sqlite >=3.33.0,<4.0a0", - "alembic", - "zlib >=1.2.11,<1.3.0a0", - "python-dateutil", - "nbformat", - "jupyter_core", - ], - ) - assert res[0] - - with suppress_output(): - solver = mamba_solver_factory(("conda-forge", "defaults"), "linux-64") - res = solver.solve( - [ - "gdal >=2.1.0", - "ncurses >=6.2,<7.0a0", - "geopandas", - "scikit-image >=0.16.0", - "pandas", - "pyproj >=2.2.0", - "libffi >=3.2.1,<4.0a0", - "six", - "tk >=8.6.10,<8.7.0a0", - "spectral", - "zlib >=1.2.11,<1.3.0a0", - "shapely", - "readline >=8.0,<9.0a0", - "python >=3.8,<3.9.0a0", - "numpy", - "python_abi 3.8.* *_cp38", - "xz >=5.2.5,<6.0a0", - "openssl >=1.1.1h,<1.1.2a", - "sqlite >=3.33.0,<4.0a0", - ], - ) - assert res[0] + assert not is_recipe_solvable(feedstock_dir, solver=solver, verbosity=VERB)[0] @flaky -def test_arrow_solvable_timeout(tmp_path): +def test_arrow_solvable_timeout(tmp_path, solver): feedstock_dir = clone_and_checkout_repo( tmp_path, "https://github.com/conda-forge/arrow-cpp-feedstock", @@ -599,6 +431,8 @@ def test_arrow_solvable_timeout(tmp_path): solvable, errors, solvable_by_variant = is_recipe_solvable( feedstock_dir, timeout=10, + solver=solver, + verbosity=VERB, ) assert solvable assert errors == [] @@ -607,7 +441,7 @@ def test_arrow_solvable_timeout(tmp_path): @flaky @pytest.mark.xfail -def test_pillow_solvable(tmp_path): +def test_pillow_solvable(tmp_path, solver): """pillow acted up for python310""" feedstock_dir = clone_and_checkout_repo( tmp_path, @@ -670,7 +504,11 @@ def test_pillow_solvable(tmp_path): check=True, ) - solvable, errors, solvable_by_variant = is_recipe_solvable(feedstock_dir) + solvable, errors, solvable_by_variant = is_recipe_solvable( + feedstock_dir, + solver=solver, + verbosity=VERB, + ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) assert any("python3.10" in k for k in solvable_by_variant) diff --git a/tests/test_solvers.py b/tests/test_solvers.py new file mode 100644 index 0000000..0671a54 --- /dev/null +++ b/tests/test_solvers.py @@ -0,0 +1,225 @@ +from flaky import flaky +import pprint +import pytest + +from conda_forge_feedstock_check_solvable.virtual_packages import virtual_package_repodata +from conda_forge_feedstock_check_solvable.rattler_solver import rattler_solver_factory +from conda_forge_feedstock_check_solvable.mamba_solver import mamba_solver_factory +from conda_forge_feedstock_check_solvable.utils import apply_pins, suppress_output + + +@pytest.fixture(scope="session", params=["rattler", "mamba"]) +def solver_factory(request): + if request.param == "mamba": + yield mamba_solver_factory + elif request.param == "rattler": + yield rattler_solver_factory + else: + raise ValueError(f"Unknown solver {request.param}") + + +@flaky +def test_solvers_apply_pins(tmp_path, solver_factory): + with open(tmp_path / "meta.yaml", "w") as fp: + fp.write( + """\ +{% set name = "cf-autotick-bot-test-package" %} +{% set version = "0.9" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + path: . + +build: + number: 8 + +requirements: + host: + - python + - pip + - jpeg + run: + - python + +test: + commands: + - echo "works!" + +about: + home: https://github.com/regro/cf-scripts + license: BSD-3-Clause + license_family: BSD + license_file: LICENSE + summary: testing feedstock for the regro-cf-autotick-bot + +extra: + recipe-maintainers: + - beckermr + - conda-forge/bot +""", + ) + + with open(tmp_path / "conda_build_config.yaml", "w") as fp: + fp.write( + """\ +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython +""", + ) + import conda_build.api + + with suppress_output(): + config = conda_build.config.get_or_merge_config( + None, + platform="linux", + arch="64", + variant_config_files=[], + ) + cbc, _ = conda_build.variants.get_package_combined_spec( + str(tmp_path), + config=config, + ) + + solver = solver_factory(("conda-forge", "defaults"), "linux-64") + + metas = conda_build.api.render( + str(tmp_path), + platform="linux", + arch="64", + ignore_system_variants=True, + variants=cbc, + permit_undefined_jinja=True, + finalize=False, + bypass_env_check=True, + channel_urls=("conda-forge", "defaults"), + ) + + m = metas[0][0] + outnames = [m.name() for m, _, _ in metas] + build_req = m.get_value("requirements/build", []) + host_req = m.get_value("requirements/host", []) + run_req = m.get_value("requirements/run", []) + _, _, build_req, rx = solver.solve(build_req, get_run_exports=True) + print("build req: %s" % pprint.pformat(build_req)) + print("build rex: %s" % pprint.pformat(rx)) + host_req = list(set(host_req) | rx["strong"]) + run_req = list(set(run_req) | rx["strong"]) + _, _, host_req, rx = solver.solve(host_req, get_run_exports=True) + print("host req: %s" % pprint.pformat(host_req)) + print("host rex: %s" % pprint.pformat(rx)) + run_req = list(set(run_req) | rx["weak"]) + run_req = apply_pins(run_req, host_req, build_req, outnames, m) + print("run req: %s" % pprint.pformat(run_req)) + assert any(r.startswith("python >=3.8") for r in run_req) + assert any(r.startswith("jpeg >=") for r in run_req) + + +@flaky +def test_solvers_constraints(solver_factory): + with suppress_output(): + solver = solver_factory(("conda-forge",), "osx-64") + solvable, err, solution = solver.solve( + ["simplejson"], constraints=["python=3.10", "zeromq=4.2"] + ) + assert solvable, err + python = [pkg for pkg in solution if pkg.split()[0] == "python"][0] + name, version, build = python.split(None, 2) + assert version.startswith("3.10.") + assert not any(pkg.startswith("zeromq") for pkg in solution), pprint.pformat( + solution + ) + + +@flaky +def test_solvers_constraints_unsolvable(solver_factory): + with suppress_output(): + solver = solver_factory(("conda-forge",), "osx-64") + solvable, err, solution = solver.solve( + ["simplejson"], constraints=["python=3.10", "python=3.11"] + ) + assert not solvable, pprint.pformat(solution) + + +@flaky +def test_solvers_nvcc_with_virtual_package(solver_factory): + with suppress_output(): + virtual_packages = virtual_package_repodata() + solver = solver_factory((virtual_packages, "conda-forge", "defaults"), "linux-64") + out = solver.solve( + ["gcc_linux-64 7.*", "gxx_linux-64 7.*", "nvcc_linux-64 11.0.*"] + ) + print(out) + assert out[0], out[1] + + +@flaky +def test_solvers_hang(solver_factory): + with suppress_output(): + solver = solver_factory(("conda-forge", "defaults"), "osx-64") + res = solver.solve( + [ + "pytest", + "selenium", + "requests-mock", + "ncurses >=6.2,<7.0a0", + "libffi >=3.2.1,<4.0a0", + "xz >=5.2.5,<6.0a0", + "nbconvert >=5.6", + "sqlalchemy", + "jsonschema", + "six >=1.11", + "python_abi 3.9.* *_cp39", + "tornado", + "jupyter", + "requests", + "jupyter_client", + "notebook >=4.2", + "tk >=8.6.10,<8.7.0a0", + "openssl >=1.1.1h,<1.1.2a", + "readline >=8.0,<9.0a0", + "fuzzywuzzy", + "python >=3.9,<3.10.0a0", + "traitlets", + "sqlite >=3.33.0,<4.0a0", + "alembic", + "zlib >=1.2.11,<1.3.0a0", + "python-dateutil", + "nbformat", + "jupyter_core", + ], + ) + assert res[0] + + with suppress_output(): + solver = solver_factory(("conda-forge", "defaults"), "linux-64") + res = solver.solve( + [ + "gdal >=2.1.0", + "ncurses >=6.2,<7.0a0", + "geopandas", + "scikit-image >=0.16.0", + "pandas", + "pyproj >=2.2.0", + "libffi >=3.2.1,<4.0a0", + "six", + "tk >=8.6.10,<8.7.0a0", + "spectral", + "zlib >=1.2.11,<1.3.0a0", + "shapely", + "readline >=8.0,<9.0a0", + "python >=3.8,<3.9.0a0", + "numpy", + "python_abi 3.8.* *_cp38", + "xz >=5.2.5,<6.0a0", + "openssl >=1.1.1h,<1.1.2a", + "sqlite >=3.33.0,<4.0a0", + ], + ) + assert res[0] From 79f122fcd28cb9453a79e7c62a2258ecf7a04132 Mon Sep 17 00:00:00 2001 From: beckermr Date: Sat, 1 Jun 2024 23:25:15 -0500 Subject: [PATCH 04/35] STY blacken --- tests/test_solvers.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_solvers.py b/tests/test_solvers.py index 0671a54..452b660 100644 --- a/tests/test_solvers.py +++ b/tests/test_solvers.py @@ -1,11 +1,14 @@ -from flaky import flaky import pprint + import pytest +from flaky import flaky -from conda_forge_feedstock_check_solvable.virtual_packages import virtual_package_repodata -from conda_forge_feedstock_check_solvable.rattler_solver import rattler_solver_factory from conda_forge_feedstock_check_solvable.mamba_solver import mamba_solver_factory +from conda_forge_feedstock_check_solvable.rattler_solver import rattler_solver_factory from conda_forge_feedstock_check_solvable.utils import apply_pins, suppress_output +from conda_forge_feedstock_check_solvable.virtual_packages import ( + virtual_package_repodata, +) @pytest.fixture(scope="session", params=["rattler", "mamba"]) @@ -151,7 +154,9 @@ def test_solvers_constraints_unsolvable(solver_factory): def test_solvers_nvcc_with_virtual_package(solver_factory): with suppress_output(): virtual_packages = virtual_package_repodata() - solver = solver_factory((virtual_packages, "conda-forge", "defaults"), "linux-64") + solver = solver_factory( + (virtual_packages, "conda-forge", "defaults"), "linux-64" + ) out = solver.solve( ["gcc_linux-64 7.*", "gxx_linux-64 7.*", "nvcc_linux-64 11.0.*"] ) From 3cd65b48aa354fee06cf8ea106df3f4b4126f46b Mon Sep 17 00:00:00 2001 From: beckermr Date: Sat, 1 Jun 2024 23:36:26 -0500 Subject: [PATCH 05/35] BUG need module --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index da460e4..2b23ce2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,5 @@ conda-forge-metadata>=0.2.0 wurlitzer requests zstandard -boltons >=23.0.0 +boltons>=23.0.0 +py-rattler=0.6 From 2594a7b6a0caefae32115b5ef57ec0575c1394ba Mon Sep 17 00:00:00 2001 From: beckermr Date: Sat, 1 Jun 2024 23:37:38 -0500 Subject: [PATCH 06/35] BUG need module --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2b23ce2..cbe1c6f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,4 @@ wurlitzer requests zstandard boltons>=23.0.0 -py-rattler=0.6 +py-rattler~=0.6 From c6c3897cc036d88f0a961b2d179078d828950831 Mon Sep 17 00:00:00 2001 From: beckermr Date: Sat, 1 Jun 2024 23:39:12 -0500 Subject: [PATCH 07/35] BUG need module --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cbe1c6f..85fc582 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,4 @@ wurlitzer requests zstandard boltons>=23.0.0 -py-rattler~=0.6 +py-rattler==0.6.* From 45f26f026b2fe4a2ffc8901c4b4b9860d584021f Mon Sep 17 00:00:00 2001 From: beckermr Date: Sun, 2 Jun 2024 01:06:51 -0500 Subject: [PATCH 08/35] TST do not timeout --- tests/test_check_solvable.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/tests/test_check_solvable.py b/tests/test_check_solvable.py index 7e637b6..401736d 100644 --- a/tests/test_check_solvable.py +++ b/tests/test_check_solvable.py @@ -77,7 +77,9 @@ def test_is_recipe_solvable_ok(feedstock_dir, solver): - conda-forge/bot """, ) - assert is_recipe_solvable(feedstock_dir, solver=solver, verbosity=VERB)[0] + assert is_recipe_solvable( + feedstock_dir, solver=solver, verbosity=VERB, timeout=None + )[0] @flaky @@ -129,6 +131,7 @@ def test_unsolvable_for_particular_python(feedstock_dir, solver): feedstock_dir, solver=solver, verbosity=VERB, + timeout=None, ) print(solvable_by_variant) assert not solvable @@ -152,6 +155,7 @@ def test_r_base_cross_solvable(solver): build_platform={"osx_arm64": "osx_64"}, solver=solver, verbosity=VERB, + timeout=None, ) assert solvable, pprint.pformat(errors) @@ -160,7 +164,10 @@ def test_r_base_cross_solvable(solver): def test_xgboost_solvable(solver): feedstock_dir = os.path.join(os.path.dirname(__file__), "xgboost-feedstock") solvable, errors, _ = is_recipe_solvable( - feedstock_dir, solver=solver, verbosity=VERB + feedstock_dir, + solver=solver, + verbosity=VERB, + timeout=None, ) assert solvable, pprint.pformat(errors) @@ -169,7 +176,10 @@ def test_xgboost_solvable(solver): def test_pandas_solvable(solver): feedstock_dir = os.path.join(os.path.dirname(__file__), "pandas-feedstock") solvable, errors, _ = is_recipe_solvable( - feedstock_dir, solver=solver, verbosity=VERB + feedstock_dir, + solver=solver, + verbosity=VERB, + timeout=None, ) assert solvable, pprint.pformat(errors) @@ -193,6 +203,7 @@ def test_arrow_solvable(tmp_path, solver): feedstock_dir, solver=solver, verbosity=VERB, + timeout=None, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -210,6 +221,7 @@ def test_guiqwt_solvable(tmp_path, solver): feedstock_dir, solver=solver, verbosity=VERB, + timeout=None, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -227,6 +239,7 @@ def test_datalad_solvable(tmp_path, solver): feedstock_dir, solver=solver, verbosity=VERB, + timeout=None, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -244,6 +257,7 @@ def test_grpcio_solvable(tmp_path, solver): feedstock_dir, solver=solver, verbosity=VERB, + timeout=None, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -266,6 +280,7 @@ def test_cupy_solvable(tmp_path, solver): feedstock_dir, solver=solver, verbosity=VERB, + timeout=None, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -319,9 +334,9 @@ def test_run_exports_constrains_conflict(feedstock_dir, tmp_path_factory, solver solvable, errors, solve_by_variant = is_recipe_solvable( feedstock_dir, additional_channels=[repodata.channel_url], - timeout=None, solver=solver, verbosity=VERB, + timeout=None, ) assert solvable, pprint.pformat(errors) @@ -366,6 +381,7 @@ def test_run_exports_constrains_notok(feedstock_dir, tmp_path_factory, solver): feedstock_dir, solver=solver, verbosity=VERB, + timeout=None, ) assert not solvable, pprint.pformat(errors) @@ -415,7 +431,9 @@ def test_is_recipe_solvable_notok(feedstock_dir, solver): - conda-forge/bot """, ) - assert not is_recipe_solvable(feedstock_dir, solver=solver, verbosity=VERB)[0] + assert not is_recipe_solvable( + feedstock_dir, solver=solver, verbosity=VERB, timeout=None + )[0] @flaky @@ -508,6 +526,7 @@ def test_pillow_solvable(tmp_path, solver): feedstock_dir, solver=solver, verbosity=VERB, + timeout=None, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) From 50bf950a6f55b0a2d139648dcaa41b5772ae5b93 Mon Sep 17 00:00:00 2001 From: beckermr Date: Sun, 2 Jun 2024 06:22:25 -0500 Subject: [PATCH 09/35] REF use solver timeout if we have it --- .github/workflows/tests.yml | 4 +- .../check_solvable.py | 22 ++++++-- .../mamba_solver.py | 6 +++ .../rattler_solver.py | 50 ++++++++++++++++++- 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 71e7d36..7df8e4d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,7 @@ jobs: shell: bash -l {0} run: | mamba install --yes --file=requirements.txt - mamba install --yes pytest flake8 flaky pip python-build setuptools_scm>=7 setuptools>=45 toml + mamba install --yes pytest flaky pip python-build setuptools_scm>=7 setuptools>=45 toml pip install -e . - name: test versions @@ -66,4 +66,4 @@ jobs: - name: test shell: bash -l {0} run: | - pytest -vvs tests + pytest -vv --durations=0 tests diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 2a0e1a3..e92d25f 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -1,5 +1,6 @@ import glob import os +import time from typing import Dict, List, Tuple import conda_build.api @@ -81,7 +82,7 @@ def is_recipe_solvable( solvable_by_variant : dict A lookup by variant config that shows if a particular config is solvable """ - if timeout: + if timeout and solver in ["mamba"]: from multiprocessing import Pipe, Process parent_conn, child_conn = Pipe() @@ -129,6 +130,7 @@ def is_recipe_solvable( build_platform=build_platform, verbosity=verbosity, solver=solver, + timeout=timeout, ) return res @@ -140,8 +142,10 @@ def _is_recipe_solvable( build_platform=None, verbosity=1, solver="mamba", + timeout=None, ) -> Tuple[bool, List[str], Dict[str, bool]]: conda_forge_feedstock_check_solvable.utils.VERBOSITY = verbosity + start_time = time.time() build_platform = build_platform or {} @@ -172,6 +176,10 @@ def _is_recipe_solvable( solvable = True solvable_by_cbc = {} for cbc_fname in cbcs: + if timeout and time.time() - start_time > timeout: + print_warning("SOLVER TIMEOUT for %s", feedstock_dir) + return True, [], {} + # we need to extract the platform (e.g., osx, linux) and arch (e.g., 64, aarm64) # conda smithy forms a string that is # @@ -196,6 +204,7 @@ def _is_recipe_solvable( ), additional_channels=additional_channels, solver_backend=solver, + timeout=timeout, ) solvable = solvable and _solvable cbc_name = os.path.basename(cbc_fname).rsplit(".", maxsplit=1)[0] @@ -215,6 +224,7 @@ def _is_recipe_solvable_on_platform( build_platform_arch=None, additional_channels=(), solver_backend="mamba", + timeout=None, ): # parse the channel sources from the CBC parser = YAML(typ="jinja2") @@ -327,6 +337,7 @@ def _is_recipe_solvable_on_platform( get_run_exports=True, ignore_run_exports_from=ign_runex_from, ignore_run_exports=ign_runex, + timeout=timeout, ) solvable = solvable and _solvable if _err is not None: @@ -359,6 +370,7 @@ def _is_recipe_solvable_on_platform( get_run_exports=True, ignore_run_exports_from=ign_runex_from, ignore_run_exports=ign_runex, + timeout=timeout, ) solvable = solvable and _solvable if _err is not None: @@ -382,7 +394,9 @@ def _is_recipe_solvable_on_platform( if run_req: run_req = apply_pins(run_req, host_req or [], build_req or [], outnames, m) run_req = remove_reqs_by_name(run_req, outnames) - _solvable, _err, _ = solver.solve(run_req, constraints=run_constrained) + _solvable, _err, _ = solver.solve( + run_req, constraints=run_constrained, timeout=timeout + ) solvable = solvable and _solvable if _err is not None: errors.append(_err) @@ -394,7 +408,9 @@ def _is_recipe_solvable_on_platform( ) if tst_req: tst_req = remove_reqs_by_name(tst_req, outnames) - _solvable, _err, _ = solver.solve(tst_req, constraints=run_constrained) + _solvable, _err, _ = solver.solve( + tst_req, constraints=run_constrained, timeout=timeout + ) solvable = solvable and _solvable if _err is not None: errors.append(_err) diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index 93e6054..e547f89 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -80,6 +80,7 @@ def solve( ignore_run_exports_from=None, ignore_run_exports=None, constraints=None, + timeout=None, ) -> Tuple[bool, List[str]]: """Solve given a set of specs. @@ -98,6 +99,8 @@ def solve( constraints : list, optional A list of package specs to apply as constraints to the solve. These packages are not included in the solution. + timeout : int, optional + Ignored by mamba. Returns ------- @@ -111,6 +114,9 @@ def solve( A dictionary with the weak and strong run exports for the packages. Only returned if get_run_exports is True. """ + if timeout is not None: + raise RuntimeError("The `timeout` keyword is not supported by mamba!") + ignore_run_exports_from = ignore_run_exports_from or [] ignore_run_exports = ignore_run_exports or [] diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index 597c44c..536c063 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -16,6 +16,21 @@ class RattlerSolver: + """Run the rattler solver (resolvo). + + Parameters + ---------- + channels : list of str + A list of the channels (e.g., `[conda-forge]`, etc.) + platform : str + The platform to be used (e.g., `linux-64`). + + Example + ------- + >>> solver = RattlerSolver(['conda-forge', 'conda-forge'], "linux-64") + >>> solver.solve(["xtensor 0.18"]) + """ + def __init__(self, channels, platform_arch) -> None: _channels = [] for c in channels: @@ -36,7 +51,40 @@ def solve( ignore_run_exports_from: List[str] = None, ignore_run_exports: List[str] = None, constraints=None, + timeout: int | None = None, ): + """Solve given a set of specs. + + Parameters + ---------- + specs : list of str + A list of package specs. You can use `conda.models.match_spec.MatchSpec` + to get them to the right form by calling + `MatchSpec(mypec).conda_build_form()` + get_run_exports : bool, optional + If True, return run exports else do not. + ignore_run_exports_from : list, optional + A list of packages from which to ignore the run exports. + ignore_run_exports : list, optional + A list of things that should be ignore in the run exports. + constraints : list, optional + A list of package specs to apply as constraints to the solve. + These packages are not included in the solution. + timeout : int, optional + The time in seconds to wait for the solver to finish before giving up. + + Returns + ------- + solvable : bool + True if the set of specs has a solution, False otherwise. + err : str + The errors as a string. If no errors, is None. + solution : list of str + A list of concrete package specs for the env. + run_exports : dict of list of str + A dictionary with the weak and strong run exports for the packages. + Only returned if get_run_exports is True. + """ ignore_run_exports_from = ignore_run_exports_from or [] ignore_run_exports = ignore_run_exports or [] success = False @@ -55,7 +103,7 @@ def solve( channels=self.channels, specs=_specs, platforms=self.platforms, - # virtual_packages=self.virtual_packages, + timeout=timeout, ) ) success = True From d8fab2992e39308bd97e8b1e512ea4c8f1bf685e Mon Sep 17 00:00:00 2001 From: beckermr Date: Sun, 2 Jun 2024 06:43:15 -0500 Subject: [PATCH 10/35] BUG rattler timeouts not working --- .../check_solvable.py | 2 +- .../rattler_solver.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index e92d25f..74a8e1f 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -82,7 +82,7 @@ def is_recipe_solvable( solvable_by_variant : dict A lookup by variant config that shows if a particular config is solvable """ - if timeout and solver in ["mamba"]: + if timeout and solver in ["mamba", "rattler"]: from multiprocessing import Pipe, Process parent_conn, child_conn = Pipe() diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index 536c063..56e5712 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -32,6 +32,7 @@ class RattlerSolver: """ def __init__(self, channels, platform_arch) -> None: + self.channels = channels _channels = [] for c in channels: if c == "defaults": @@ -40,9 +41,9 @@ def __init__(self, channels, platform_arch) -> None: _channels.append("https://repo.anaconda.com/pkgs/msys2") else: _channels.append(c) - self.channels = [Channel(c) for c in _channels] + self._channels = [Channel(c) for c in _channels] self.platform_arch = platform_arch - self.platforms = [Platform(self.platform_arch), Platform("noarch")] + self._platforms = [Platform(self.platform_arch), Platform("noarch")] def solve( self, @@ -85,6 +86,9 @@ def solve( A dictionary with the weak and strong run exports for the packages. Only returned if get_run_exports is True. """ + if timeout is not None: + raise RuntimeError("The `timeout` keyword is currently ignored!") + ignore_run_exports_from = ignore_run_exports_from or [] ignore_run_exports = ignore_run_exports or [] success = False @@ -100,10 +104,10 @@ def solve( solution = asyncio.run( solve( - channels=self.channels, + channels=self._channels, specs=_specs, - platforms=self.platforms, - timeout=timeout, + platforms=self._platforms, + timeout=None, ) ) success = True @@ -125,7 +129,7 @@ def solve( print_warning( "RATTLER failed to solve specs \n\n%s\n\nfor channels " "\n\n%s\n\nThe reported errors are:\n\n%s\n", - pprint.pformat(_specs), + pprint.pformat(specs), pprint.pformat(self.channels), err, ) From a8796cf92835849912269b56e92436a662fd955d Mon Sep 17 00:00:00 2001 From: beckermr Date: Sun, 2 Jun 2024 07:25:57 -0500 Subject: [PATCH 11/35] FIXME ignore timeout for now --- .../check_solvable.py | 2 +- .../rattler_solver.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 74a8e1f..e92d25f 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -82,7 +82,7 @@ def is_recipe_solvable( solvable_by_variant : dict A lookup by variant config that shows if a particular config is solvable """ - if timeout and solver in ["mamba", "rattler"]: + if timeout and solver in ["mamba"]: from multiprocessing import Pipe, Process parent_conn, child_conn = Pipe() diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index 56e5712..1d52819 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -1,5 +1,6 @@ import asyncio import copy +import datetime import os import pprint from typing import List @@ -86,8 +87,12 @@ def solve( A dictionary with the weak and strong run exports for the packages. Only returned if get_run_exports is True. """ + if timeout is not None: - raise RuntimeError("The `timeout` keyword is currently ignored!") + print_warning( + "The `timeout` keyword is currently buggy in rattler and so is being ignored!" + ) + timeout = None ignore_run_exports_from = ignore_run_exports_from or [] ignore_run_exports = ignore_run_exports or [] @@ -102,12 +107,15 @@ def solve( "RATTLER running solver for specs \n\n%s\n", pprint.pformat(_specs) ) + if timeout is not None: + timeout = datetime.timedelta(seconds=timeout) + solution = asyncio.run( solve( channels=self._channels, specs=_specs, platforms=self._platforms, - timeout=None, + timeout=timeout, ) ) success = True From 11ca9de8238df6eed9a30e31e473317832136131 Mon Sep 17 00:00:00 2001 From: beckermr Date: Sun, 2 Jun 2024 08:33:59 -0500 Subject: [PATCH 12/35] REF use context manager for env --- .../check_solvable.py | 120 +++++++++--------- conda_forge_feedstock_check_solvable/utils.py | 13 ++ 2 files changed, 73 insertions(+), 60 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index e92d25f..2f613cd 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -14,6 +14,7 @@ MAX_GLIBC_MINOR, apply_pins, get_run_exports, + override_env_var, print_debug, print_info, print_warning, @@ -151,67 +152,66 @@ def _is_recipe_solvable( additional_channels = additional_channels or [] additional_channels += [virtual_package_repodata()] - os.environ["CONDA_OVERRIDE_GLIBC"] = "2.%d" % MAX_GLIBC_MINOR - - errors = [] - cbcs = sorted(glob.glob(os.path.join(feedstock_dir, ".ci_support", "*.yaml"))) - if len(cbcs) == 0: - errors.append( - "No `.ci_support/*.yaml` files found! This can happen when a rerender " - "results in no builds for a recipe (e.g., a recipe is python 2.7 only). " - "This attempted migration is being reported as not solvable.", - ) - print_warning(errors[-1]) - return False, errors, {} - - if not os.path.exists(os.path.join(feedstock_dir, "recipe", "meta.yaml")): - errors.append( - "No `recipe/meta.yaml` file found! This issue is quite weird and " - "someone should investigate!", - ) - print_warning(errors[-1]) - return False, errors, {} - - print_info("CHECKING FEEDSTOCK: %s", os.path.basename(feedstock_dir)) - solvable = True - solvable_by_cbc = {} - for cbc_fname in cbcs: - if timeout and time.time() - start_time > timeout: - print_warning("SOLVER TIMEOUT for %s", feedstock_dir) - return True, [], {} - - # we need to extract the platform (e.g., osx, linux) and arch (e.g., 64, aarm64) - # conda smithy forms a string that is - # - # {{ platform }} if arch == 64 - # {{ platform }}_{{ arch }} if arch != 64 - # - # Thus we undo that munging here. - _parts = os.path.basename(cbc_fname).split("_") - platform = _parts[0] - arch = _parts[1] - if arch not in ["32", "aarch64", "ppc64le", "armv7l", "arm64"]: - arch = "64" - - print_info("CHECKING RECIPE SOLVABLE: %s", os.path.basename(cbc_fname)) - _solvable, _errors = _is_recipe_solvable_on_platform( - os.path.join(feedstock_dir, "recipe"), - cbc_fname, - platform, - arch, - build_platform_arch=( - build_platform.get(f"{platform}_{arch}", f"{platform}_{arch}") - ), - additional_channels=additional_channels, - solver_backend=solver, - timeout=timeout, - ) - solvable = solvable and _solvable - cbc_name = os.path.basename(cbc_fname).rsplit(".", maxsplit=1)[0] - errors.extend([f"{cbc_name}: {e}" for e in _errors]) - solvable_by_cbc[cbc_name] = _solvable + with override_env_var("CONDA_OVERRIDE_GLIBC", "2.%d" % MAX_GLIBC_MINOR): + errors = [] + cbcs = sorted(glob.glob(os.path.join(feedstock_dir, ".ci_support", "*.yaml"))) + if len(cbcs) == 0: + errors.append( + "No `.ci_support/*.yaml` files found! This can happen when a rerender " + "results in no builds for a recipe (e.g., a recipe is python 2.7 only). " + "This attempted migration is being reported as not solvable.", + ) + print_warning(errors[-1]) + return False, errors, {} - del os.environ["CONDA_OVERRIDE_GLIBC"] + if not os.path.exists(os.path.join(feedstock_dir, "recipe", "meta.yaml")): + errors.append( + "No `recipe/meta.yaml` file found! This issue is quite weird and " + "someone should investigate!", + ) + print_warning(errors[-1]) + return False, errors, {} + + print_info("CHECKING FEEDSTOCK: %s", os.path.basename(feedstock_dir)) + solvable = True + solvable_by_cbc = {} + for cbc_fname in cbcs: + time_left = timeout - (time.time() - start_time) if timeout else None + if timeout and time_left <= 0: + print_warning("SOLVER TIMEOUT for %s", feedstock_dir) + return True, [], {} + + # we need to extract the platform (e.g., osx, linux) and arch (e.g., 64, aarm64) + # conda smithy forms a string that is + # + # {{ platform }} if arch == 64 + # {{ platform }}_{{ arch }} if arch != 64 + # + # Thus we undo that munging here. + _parts = os.path.basename(cbc_fname).split("_") + platform = _parts[0] + arch = _parts[1] + if arch not in ["32", "aarch64", "ppc64le", "armv7l", "arm64"]: + arch = "64" + + print_info("CHECKING RECIPE SOLVABLE: %s", os.path.basename(cbc_fname)) + + _solvable, _errors = _is_recipe_solvable_on_platform( + os.path.join(feedstock_dir, "recipe"), + cbc_fname, + platform, + arch, + build_platform_arch=( + build_platform.get(f"{platform}_{arch}", f"{platform}_{arch}") + ), + additional_channels=additional_channels, + solver_backend=solver, + timeout=time_left, + ) + solvable = solvable and _solvable + cbc_name = os.path.basename(cbc_fname).rsplit(".", maxsplit=1)[0] + errors.extend([f"{cbc_name}: {e}" for e in _errors]) + solvable_by_cbc[cbc_name] = _solvable return solvable, errors, solvable_by_cbc diff --git a/conda_forge_feedstock_check_solvable/utils.py b/conda_forge_feedstock_check_solvable/utils.py index c9ec082..635a934 100644 --- a/conda_forge_feedstock_check_solvable/utils.py +++ b/conda_forge_feedstock_check_solvable/utils.py @@ -134,6 +134,19 @@ def print_debug(fmt, *args): print_verb(fmt, *args, verbosity=3) +@contextlib.contextmanager +def override_env_var(name, value): + old_value = os.environ.get(name, None) + try: + os.environ[name] = value + yield + finally: + if old_value is None: + del os.environ[name] + else: + os.environ[name] = old_value + + @contextlib.contextmanager def suppress_output(): if "CONDA_FORGE_FEEDSTOCK_CHECK_SOLVABLE_DEBUG" in os.environ: From 7b1ec3b6a5d1877a9c0b3879fa4904c9d58b0e1a Mon Sep 17 00:00:00 2001 From: beckermr Date: Mon, 3 Jun 2024 07:04:34 -0500 Subject: [PATCH 13/35] ENH add constraints and remove hack on times --- conda_forge_feedstock_check_solvable/rattler_solver.py | 9 ++------- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index 1d52819..205666d 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -87,13 +87,6 @@ def solve( A dictionary with the weak and strong run exports for the packages. Only returned if get_run_exports is True. """ - - if timeout is not None: - print_warning( - "The `timeout` keyword is currently buggy in rattler and so is being ignored!" - ) - timeout = None - ignore_run_exports_from = ignore_run_exports_from or [] ignore_run_exports = ignore_run_exports or [] success = False @@ -102,6 +95,7 @@ def solve( try: _specs = [MatchSpec(s) for s in specs] + _constraints = [MatchSpec(c) for c in constraints] if constraints else None print_debug( "RATTLER running solver for specs \n\n%s\n", pprint.pformat(_specs) @@ -116,6 +110,7 @@ def solve( specs=_specs, platforms=self._platforms, timeout=timeout, + constraints=_constraints, ) ) success = True diff --git a/requirements.txt b/requirements.txt index 85fc582..3b70167 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,4 @@ wurlitzer requests zstandard boltons>=23.0.0 -py-rattler==0.6.* +py-rattler>=0.6.2,<0.7a0 From cb5838ea1760b2dee01044da8016f3f2a69e7b4c Mon Sep 17 00:00:00 2001 From: beckermr Date: Mon, 3 Jun 2024 15:30:14 -0500 Subject: [PATCH 14/35] BUG try and see what pins are in the local env --- .github/workflows/tests.yml | 3 +++ tests/conftest.py | 19 +++++++++++++++++++ tests/test_check_solvable.py | 7 +------ tests/test_solvers.py | 13 ------------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7df8e4d..5fbe8e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,4 +66,7 @@ jobs: - name: test shell: bash -l {0} run: | + conda config --show pinned_packages + cat "${CONDA_PREFIX}/conda-meta/pinned" + pytest -vv --durations=0 tests diff --git a/tests/conftest.py b/tests/conftest.py index a2f39af..48ff8ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,8 +4,12 @@ import pytest +from conda_forge_feedstock_check_solvable.mamba_solver import mamba_solver_factory +from conda_forge_feedstock_check_solvable.rattler_solver import rattler_solver_factory + FEEDSTOCK_DIR = os.path.join(os.path.dirname(__file__), "test_feedstock") +ALL_SOLVERS = ["rattler", "mamba"] @pytest.fixture() def feedstock_dir(tmp_path): @@ -15,3 +19,18 @@ def feedstock_dir(tmp_path): for fn in os.listdir(src_ci_support): shutil.copy(src_ci_support / fn, ci_support / fn) return str(tmp_path) + + +@pytest.fixture(scope="session", params=ALL_SOLVERS) +def solver(request): + yield request.param + + +@pytest.fixture(scope="session", params=ALL_SOLVERS) +def solver_factory(request): + if request.param == "mamba": + yield mamba_solver_factory + elif request.param == "rattler": + yield rattler_solver_factory + else: + raise ValueError(f"Unknown solver {request.param}") diff --git a/tests/test_check_solvable.py b/tests/test_check_solvable.py index adfa2b3..ae64e1e 100644 --- a/tests/test_check_solvable.py +++ b/tests/test_check_solvable.py @@ -17,11 +17,6 @@ VERB = 1 -@pytest.fixture(scope="session", params=["rattler", "mamba"]) -def solver(request): - yield request.param - - @flaky def test_is_recipe_solvable_ok(feedstock_dir, solver): recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") @@ -437,7 +432,7 @@ def test_arrow_solvable_timeout(tmp_path, solver): for _ in range(6): solvable, errors, solvable_by_variant = is_recipe_solvable( feedstock_dir, - timeout=10, + timeout=0.1, solver=solver, verbosity=VERB, ) diff --git a/tests/test_solvers.py b/tests/test_solvers.py index 452b660..5609e77 100644 --- a/tests/test_solvers.py +++ b/tests/test_solvers.py @@ -1,26 +1,13 @@ import pprint -import pytest from flaky import flaky -from conda_forge_feedstock_check_solvable.mamba_solver import mamba_solver_factory -from conda_forge_feedstock_check_solvable.rattler_solver import rattler_solver_factory from conda_forge_feedstock_check_solvable.utils import apply_pins, suppress_output from conda_forge_feedstock_check_solvable.virtual_packages import ( virtual_package_repodata, ) -@pytest.fixture(scope="session", params=["rattler", "mamba"]) -def solver_factory(request): - if request.param == "mamba": - yield mamba_solver_factory - elif request.param == "rattler": - yield rattler_solver_factory - else: - raise ValueError(f"Unknown solver {request.param}") - - @flaky def test_solvers_apply_pins(tmp_path, solver_factory): with open(tmp_path / "meta.yaml", "w") as fp: From 34bd427a8b3245bd4cc69688da22c4f3f4292605 Mon Sep 17 00:00:00 2001 From: beckermr Date: Mon, 3 Jun 2024 15:34:38 -0500 Subject: [PATCH 15/35] remove debug code --- .github/workflows/tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5fbe8e7..7df8e4d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,7 +66,4 @@ jobs: - name: test shell: bash -l {0} run: | - conda config --show pinned_packages - cat "${CONDA_PREFIX}/conda-meta/pinned" - pytest -vv --durations=0 tests From cfce17f52d779617e1e57655495c0fd208878ee8 Mon Sep 17 00:00:00 2001 From: beckermr Date: Mon, 3 Jun 2024 15:35:33 -0500 Subject: [PATCH 16/35] TST do not retry things we expect to fail --- tests/test_check_solvable.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_check_solvable.py b/tests/test_check_solvable.py index ae64e1e..2739442 100644 --- a/tests/test_check_solvable.py +++ b/tests/test_check_solvable.py @@ -441,7 +441,6 @@ def test_arrow_solvable_timeout(tmp_path, solver): assert solvable_by_variant == {} -@flaky @pytest.mark.xfail def test_pillow_solvable(tmp_path, solver): """pillow acted up for python310""" From 34ed77a00dcff3111da8b394efc7d3a8b68bc9ba Mon Sep 17 00:00:00 2001 From: beckermr Date: Mon, 3 Jun 2024 15:38:31 -0500 Subject: [PATCH 17/35] STY blacken --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 48ff8ad..c24fe4e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ ALL_SOLVERS = ["rattler", "mamba"] + @pytest.fixture() def feedstock_dir(tmp_path): ci_support = tmp_path / ".ci_support" From 54c863937663a39afa78c99e1406b5d47b426a59 Mon Sep 17 00:00:00 2001 From: beckermr Date: Mon, 3 Jun 2024 15:44:36 -0500 Subject: [PATCH 18/35] TST test solvers separately --- .github/workflows/tests.yml | 9 +++++++-- tests/conftest.py | 18 +++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7df8e4d..6e0533e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,7 +63,12 @@ jobs: python -m pip install -v --no-deps --no-build-isolation -e . - - name: test + - name: test w/ rattler shell: bash -l {0} run: | - pytest -vv --durations=0 tests + pytest -vv --durations=0 --solver=rattler tests + + - name: test w/ mamba + shell: bash -l {0} + run: | + pytest -vv --durations=0 --solver=mamba tests diff --git a/tests/conftest.py b/tests/conftest.py index c24fe4e..d4084a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,6 +12,15 @@ ALL_SOLVERS = ["rattler", "mamba"] +def pytest_addoption(parser): + parser.addoption( + "--solver", + action="append", + default=[], + help="conda solver to use", + ) + + @pytest.fixture() def feedstock_dir(tmp_path): ci_support = tmp_path / ".ci_support" @@ -24,14 +33,17 @@ def feedstock_dir(tmp_path): @pytest.fixture(scope="session", params=ALL_SOLVERS) def solver(request): - yield request.param + solvers = request.config.getoption("solver") or ALL_SOLVERS + if request.param in solvers: + yield request.param @pytest.fixture(scope="session", params=ALL_SOLVERS) def solver_factory(request): - if request.param == "mamba": + solvers = request.config.getoption("solver") or ALL_SOLVERS + if request.param == "mamba" and "mamba" in solvers: yield mamba_solver_factory - elif request.param == "rattler": + elif request.param == "rattler" and "rattler" in solvers: yield rattler_solver_factory else: raise ValueError(f"Unknown solver {request.param}") From d583a0656aaf865f8db3a9f34bd742453ddc0afa Mon Sep 17 00:00:00 2001 From: beckermr Date: Mon, 3 Jun 2024 16:01:06 -0500 Subject: [PATCH 19/35] TST fix test parametrization --- tests/conftest.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d4084a7..777aca6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,19 +31,19 @@ def feedstock_dir(tmp_path): return str(tmp_path) -@pytest.fixture(scope="session", params=ALL_SOLVERS) -def solver(request): - solvers = request.config.getoption("solver") or ALL_SOLVERS - if request.param in solvers: - yield request.param - - -@pytest.fixture(scope="session", params=ALL_SOLVERS) -def solver_factory(request): - solvers = request.config.getoption("solver") or ALL_SOLVERS - if request.param == "mamba" and "mamba" in solvers: - yield mamba_solver_factory - elif request.param == "rattler" and "rattler" in solvers: - yield rattler_solver_factory - else: - raise ValueError(f"Unknown solver {request.param}") +def pytest_generate_tests(metafunc): + if "solver" in metafunc.fixturenames: + metafunc.parametrize( + "solver", metafunc.config.getoption("solver") or ALL_SOLVERS + ) + if "solver_factory" in metafunc.fixturenames: + solvers = metafunc.config.getoption("solver") or ALL_SOLVERS + factories = [] + for solver in solvers: + if solver == "mamba": + factories.append(mamba_solver_factory) + elif solver == "rattler": + factories.append(rattler_solver_factory) + else: + raise ValueError(f"Unknown solver {solver}") + metafunc.parametrize("solver_factory", factories) From c918eafd5a4c6ffac40e97bc39c0f3184a409703 Mon Sep 17 00:00:00 2001 From: beckermr Date: Mon, 3 Jun 2024 17:55:52 -0500 Subject: [PATCH 20/35] try a longer cache time for tests --- conda_forge_feedstock_check_solvable/mamba_solver.py | 2 +- conda_forge_feedstock_check_solvable/rattler_solver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index e547f89..84c6ac2 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -212,6 +212,6 @@ def _get_run_exports( return run_exports -@cachetools.func.ttl_cache(maxsize=8, ttl=60) +@cachetools.func.ttl_cache(maxsize=8, ttl=600) def mamba_solver_factory(channels, platform): return MambaSolver(list(channels), platform) diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index 205666d..609cbc6 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -177,6 +177,6 @@ def _get_run_exports( return run_exports -@cachetools.func.ttl_cache(maxsize=8, ttl=60) +@cachetools.func.ttl_cache(maxsize=8, ttl=600) def rattler_solver_factory(channels, platform): return RattlerSolver(list(channels), platform) From c7e0a02b4db5e6a2c5ba28c30323882112ac728c Mon Sep 17 00:00:00 2001 From: beckermr Date: Wed, 5 Jun 2024 05:40:33 -0500 Subject: [PATCH 21/35] REF move to timeout timer and always cache --- .../check_solvable.py | 138 ++++++++---------- .../mamba_solver.py | 4 +- .../rattler_solver.py | 4 +- conda_forge_feedstock_check_solvable/utils.py | 24 +++ 4 files changed, 91 insertions(+), 79 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 6bf441a..f9f8e84 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -1,6 +1,5 @@ import glob import os -import time from typing import Dict, List, Tuple import conda_build.api @@ -12,6 +11,8 @@ from conda_forge_feedstock_check_solvable.rattler_solver import rattler_solver_factory from conda_forge_feedstock_check_solvable.utils import ( MAX_GLIBC_MINOR, + TimeoutTimer, + TimeoutTimerException, apply_pins, get_run_exports, override_env_var, @@ -26,22 +27,6 @@ ) -def _func(feedstock_dir, additional_channels, build_platform, verbosity, solver, conn): - try: - res = _is_recipe_solvable( - feedstock_dir, - additional_channels=additional_channels, - build_platform=build_platform, - verbosity=verbosity, - solver=solver, - ) - conn.send(res) - except Exception as e: - conn.send(e) - finally: - conn.close() - - def is_recipe_solvable( feedstock_dir, additional_channels=None, @@ -64,9 +49,12 @@ def is_recipe_solvable( additional_channels : list of str, optional If given, these channels will be used in addition to the main ones. timeout : int, optional - If not None, then the work will be run in a separate process and - this function will return True if the work doesn't complete before `timeout` - seconds. + If not None, then this function will return True if the solver checks don't + complete before `timeout` seconds. + build_platform : dict, optional + A dictionary mapping the target platform-arch to the platform-arch to use for + the build. If not given, the build platform-arch will be the same as + the target platform-arch. verbosity : int An int indicating the level of verbosity from 0 (no output) to 3 (gobbs of output). @@ -83,55 +71,21 @@ def is_recipe_solvable( solvable_by_variant : dict A lookup by variant config that shows if a particular config is solvable """ - if timeout and solver in ["mamba"]: - from multiprocessing import Pipe, Process - - parent_conn, child_conn = Pipe() - p = Process( - target=_func, - args=( - feedstock_dir, - additional_channels, - build_platform, - verbosity, - solver, - child_conn, - ), - ) - p.start() - if parent_conn.poll(timeout): - res = parent_conn.recv() - if isinstance(res, Exception): - res = ( - False, - [repr(res)], - {}, - ) - else: - print_warning("SOLVER TIMEOUT for %s", feedstock_dir) - res = ( - True, - [], - {}, - ) - - parent_conn.close() - - p.join(0) - p.terminate() - p.kill() - try: - p.close() - except ValueError: - pass - else: + try: res = _is_recipe_solvable( feedstock_dir, additional_channels=additional_channels, build_platform=build_platform, verbosity=verbosity, solver=solver, - timeout=timeout, + timeout_timer=TimeoutTimer(timeout if timeout is not None else 2e30), + ) + except TimeoutTimerException: + print_warning("SOLVER TIMEOUT for %s", feedstock_dir) + res = ( + True, + [], + {}, ) return res @@ -143,16 +97,18 @@ def _is_recipe_solvable( build_platform=None, verbosity=1, solver="mamba", - timeout=None, + timeout_timer=None, ) -> Tuple[bool, List[str], Dict[str, bool]]: conda_forge_feedstock_check_solvable.utils.VERBOSITY = verbosity - start_time = time.time() + timeout_timer = timeout_timer or TimeoutTimer(2e30) build_platform = build_platform or {} additional_channels = additional_channels or [] additional_channels += [virtual_package_repodata()] + timeout_timer.raise_for_timeout() + with override_env_var("CONDA_OVERRIDE_GLIBC", "2.%d" % MAX_GLIBC_MINOR): errors = [] cbcs = sorted(glob.glob(os.path.join(feedstock_dir, ".ci_support", "*.yaml"))) @@ -177,10 +133,7 @@ def _is_recipe_solvable( solvable = True solvable_by_cbc = {} for cbc_fname in cbcs: - time_left = timeout - (time.time() - start_time) if timeout else None - if timeout and time_left <= 0: - print_warning("SOLVER TIMEOUT for %s", feedstock_dir) - return True, [], {} + timeout_timer.raise_for_timeout() # we need to extract the platform (e.g., osx, linux) and arch (e.g., 64, aarm64) # conda smithy forms a string that is @@ -207,7 +160,7 @@ def _is_recipe_solvable( ), additional_channels=additional_channels, solver_backend=solver, - timeout=time_left, + timeout_time=timeout_timer, ) solvable = solvable and _solvable cbc_name = os.path.basename(cbc_fname).rsplit(".", maxsplit=1)[0] @@ -225,8 +178,10 @@ def _is_recipe_solvable_on_platform( build_platform_arch=None, additional_channels=(), solver_backend="mamba", - timeout=None, + timeout_timer=None, ): + timeout_timer = timeout_timer or TimeoutTimer(2e30) + # parse the channel sources from the CBC parser = YAML(typ="jinja2") parser.indent(mapping=2, sequence=4, offset=2) @@ -256,12 +211,15 @@ def _is_recipe_solvable_on_platform( arch, ) + timeout_timer.raise_for_timeout() + # here we extract the conda build config in roughly the same way that # it would be used in a real build print_debug("rendering recipe with conda build") with suppress_output(): for att in range(2): + timeout_timer.raise_for_timeout() try: if att == 1: os.system("rm -f %s/conda_build_config.yaml" % recipe_dir) @@ -281,6 +239,8 @@ def _is_recipe_solvable_on_platform( else: raise e + timeout_timer.raise_for_timeout() + # now we render the meta.yaml into an actual recipe metas = conda_build.api.render( recipe_dir, @@ -294,6 +254,8 @@ def _is_recipe_solvable_on_platform( channel_urls=channel_sources, ) + timeout_timer.raise_for_timeout() + # get build info if build_platform_arch is not None: build_platform, build_arch = build_platform_arch.split("_") @@ -312,15 +274,20 @@ def _is_recipe_solvable_on_platform( raise ValueError(f"Unknown solver backend {solver_backend}") solver = solver_factory(tuple(channel_sources), f"{platform}-{arch}") + timeout_timer.raise_for_timeout() + build_solver = solver_factory( tuple(channel_sources), f"{build_platform}-{build_arch}", ) + timeout_timer.raise_for_timeout() solvable = True errors = [] outnames = [m.name() for m, _, _ in metas] for m, _, _ in metas: + timeout_timer.raise_for_timeout() + print_debug("checking recipe %s", m.name()) build_req = m.get_value("requirements/build", []) @@ -338,8 +305,12 @@ def _is_recipe_solvable_on_platform( get_run_exports=True, ignore_run_exports_from=ign_runex_from, ignore_run_exports=ign_runex, - timeout=timeout, + timeout=timeout_timer.remaining + if solver_backend == "rattler" + else None, ) + timeout_timer.raise_for_timeout() + solvable = solvable and _solvable if _err is not None: errors.append(_err) @@ -371,8 +342,12 @@ def _is_recipe_solvable_on_platform( get_run_exports=True, ignore_run_exports_from=ign_runex_from, ignore_run_exports=ign_runex, - timeout=timeout, + timeout=timeout_timer.remaining + if solver_backend == "rattler" + else None, ) + timeout_timer.raise_for_timeout() + solvable = solvable and _solvable if _err is not None: errors.append(_err) @@ -396,8 +371,14 @@ def _is_recipe_solvable_on_platform( run_req = apply_pins(run_req, host_req or [], build_req or [], outnames, m) run_req = remove_reqs_by_name(run_req, outnames) _solvable, _err, _ = solver.solve( - run_req, constraints=run_constrained, timeout=timeout + run_req, + constraints=run_constrained, + timeout=timeout_timer.remaining + if solver_backend == "rattler" + else None, ) + timeout_timer.raise_for_timeout() + solvable = solvable and _solvable if _err is not None: errors.append(_err) @@ -410,13 +391,20 @@ def _is_recipe_solvable_on_platform( if tst_req: tst_req = remove_reqs_by_name(tst_req, outnames) _solvable, _err, _ = solver.solve( - tst_req, constraints=run_constrained, timeout=timeout + tst_req, + constraints=run_constrained, + timeout=timeout_timer.remaining + if solver_backend == "rattler" + else None, ) + timeout_timer.raise_for_timeout() + solvable = solvable and _solvable if _err is not None: errors.append(_err) print_info("RUN EXPORT CACHE STATUS: %s", get_run_exports.cache_info()) + print_info("SOLVER CACHE STATUS: %s", solver_factory.cache_info()) print_info( "SOLVER MEM USAGE: %d MB", psutil.Process().memory_info().rss // 1024**2, diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index 273a54e..af4b139 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -13,9 +13,9 @@ import copy import pprint import textwrap +from functools import lru_cache from typing import List, Tuple -import cachetools.func import libmambapy as api import rapidjson as json from conda.base.context import context @@ -215,6 +215,6 @@ def _get_run_exports( return run_exports -@cachetools.func.ttl_cache(maxsize=8, ttl=600) +@lru_cache(maxsize=128) def mamba_solver_factory(channels, platform): return MambaSolver(list(channels), platform) diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index 609cbc6..e362b80 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -3,9 +3,9 @@ import datetime import os import pprint +from functools import lru_cache from typing import List -import cachetools.func from rattler import Channel, MatchSpec, Platform, RepoDataRecord, solve from conda_forge_feedstock_check_solvable.utils import ( @@ -177,6 +177,6 @@ def _get_run_exports( return run_exports -@cachetools.func.ttl_cache(maxsize=8, ttl=600) +@lru_cache(maxsize=128) def rattler_solver_factory(channels, platform): return RattlerSolver(list(channels), platform) diff --git a/conda_forge_feedstock_check_solvable/utils.py b/conda_forge_feedstock_check_solvable/utils.py index 3ee9962..f07a75c 100644 --- a/conda_forge_feedstock_check_solvable/utils.py +++ b/conda_forge_feedstock_check_solvable/utils.py @@ -5,6 +5,7 @@ import os import subprocess import tempfile +import time import traceback from collections.abc import Mapping @@ -177,6 +178,29 @@ def suppress_output(): pass +class TimeoutTimerException(Exception): + pass + + +class TimeoutTimer: + def __init__(self, timeout, name=None): + self.timeout = timeout + self.name = name + self._start = time.monotonic() + + @property + def elapsed(self): + return time.monotonic() - self._start + + @property + def remaining(self): + return self.timeout - self.elapsed + + def raise_for_timeout(self): + if self.elapsed > self.timeout: + raise TimeoutTimerException("timeout out for %s" % self.name) + + def _munge_req_star(req): reqs = [] From 05043e9aaa089fdce4f0d486276780d5187c723f Mon Sep 17 00:00:00 2001 From: beckermr Date: Wed, 5 Jun 2024 05:43:03 -0500 Subject: [PATCH 22/35] BUG wrong keyword name --- conda_forge_feedstock_check_solvable/check_solvable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index f9f8e84..a031f6d 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -160,7 +160,7 @@ def _is_recipe_solvable( ), additional_channels=additional_channels, solver_backend=solver, - timeout_time=timeout_timer, + timeout_timer=timeout_timer, ) solvable = solvable and _solvable cbc_name = os.path.basename(cbc_fname).rsplit(".", maxsplit=1)[0] From a4f918f78846c7e088569152bc9f540abee2935a Mon Sep 17 00:00:00 2001 From: beckermr Date: Wed, 5 Jun 2024 05:46:01 -0500 Subject: [PATCH 23/35] BUG one week --- conda_forge_feedstock_check_solvable/check_solvable.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index a031f6d..48ab8e8 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -78,7 +78,7 @@ def is_recipe_solvable( build_platform=build_platform, verbosity=verbosity, solver=solver, - timeout_timer=TimeoutTimer(timeout if timeout is not None else 2e30), + timeout_timer=TimeoutTimer(timeout if timeout is not None else 6e5), ) except TimeoutTimerException: print_warning("SOLVER TIMEOUT for %s", feedstock_dir) @@ -100,7 +100,7 @@ def _is_recipe_solvable( timeout_timer=None, ) -> Tuple[bool, List[str], Dict[str, bool]]: conda_forge_feedstock_check_solvable.utils.VERBOSITY = verbosity - timeout_timer = timeout_timer or TimeoutTimer(2e30) + timeout_timer = timeout_timer or TimeoutTimer(6e5) build_platform = build_platform or {} @@ -180,7 +180,7 @@ def _is_recipe_solvable_on_platform( solver_backend="mamba", timeout_timer=None, ): - timeout_timer = timeout_timer or TimeoutTimer(2e30) + timeout_timer = timeout_timer or TimeoutTimer(6e5) # parse the channel sources from the CBC parser = YAML(typ="jinja2") From 5977a46cef5b1a800859e48b608ccec164b4d6f0 Mon Sep 17 00:00:00 2001 From: beckermr Date: Wed, 5 Jun 2024 05:47:25 -0500 Subject: [PATCH 24/35] ENH better solver error output --- .../rattler_solver.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index e362b80..8d97cd7 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -3,6 +3,7 @@ import datetime import os import pprint +import textwrap from functools import lru_cache from typing import List @@ -130,11 +131,13 @@ def solve( except Exception as e: err = str(e) print_warning( - "RATTLER failed to solve specs \n\n%s\n\nfor channels " + "MAMBA failed to solve specs \n\n%s\n\nwith " + "constraints \n\n%s\n\nfor channels " "\n\n%s\n\nThe reported errors are:\n\n%s\n", - pprint.pformat(specs), - pprint.pformat(self.channels), - err, + textwrap.indent(pprint.pformat(specs), " "), + textwrap.indent(pprint.pformat(constraints), " "), + textwrap.indent(pprint.pformat(self.channels), " "), + textwrap.indent(err, " "), ) success = False run_exports = copy.deepcopy(DEFAULT_RUN_EXPORTS) From 8d6a97ef7adcab5c75a75670406f24dd976122d0 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Wed, 5 Jun 2024 05:49:57 -0500 Subject: [PATCH 25/35] PROD remove requirement not needed --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3b70167..963a84b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ python-rapidjson requests ruamel.yaml -cachetools conda conda-package-handling conda-smithy From d7b061be6fcda71fb454d5263b3ac7eb244fa048 Mon Sep 17 00:00:00 2001 From: beckermr Date: Wed, 5 Jun 2024 07:22:07 -0500 Subject: [PATCH 26/35] BUG fixed issue with cache in mamba --- .../mamba_solver.py | 74 ++++++--- .../rattler_solver.py | 2 + conda_forge_feedstock_check_solvable/utils.py | 2 +- tests/test_solvers.py | 153 ++++++++++++++++++ 4 files changed, 208 insertions(+), 23 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index af4b139..9130e11 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -28,6 +28,7 @@ get_run_exports, print_debug, print_warning, + suppress_output, ) pkgs_dirs = context.pkgs_dirs @@ -41,6 +42,42 @@ api.Context().channel_priority = api.ChannelPriority.kStrict +def _get_pool(channels, platform, constraints): + with suppress_output(): + pool = api.Pool() + + repos = [] + load_channels( + pool, + channels, + repos, + platform=platform, + has_priority=True, + ) + for repo in repos: + # need set_installed for add_pin, not sure why + repo.set_installed() + + return pool + + +def _get_solver(channels, platform, constraints): + pool = _get_pool(channels, platform, constraints) + + solver_options = [(api.SOLVER_FLAG_ALLOW_DOWNGRADE, 1)] + solver = api.Solver(pool, solver_options) + + for constraint in constraints: + solver.add_pin(constraint) + + return solver, pool + + +@lru_cache(maxsize=128) +def _get_solver_cached(channels, platform, constraints): + return _get_solver(channels, platform, constraints) + + class MambaSolver: """Run the mamba solver. @@ -57,22 +94,10 @@ class MambaSolver: >>> solver.solve(["xtensor 0.18"]) """ - def __init__(self, channels, platform): + def __init__(self, channels, platform, _use_cache=False): self.channels = channels self.platform = platform - self.pool = api.Pool() - - self.repos = [] - self.index = load_channels( - self.pool, - self.channels, - self.repos, - platform=platform, - has_priority=True, - ) - for repo in self.repos: - # need set_installed for add_pin, not sure why - repo.set_installed() + self._use_cache = _use_cache def solve( self, @@ -121,19 +146,23 @@ def solve( ignore_run_exports_from = ignore_run_exports_from or [] ignore_run_exports = ignore_run_exports or [] - solver_options = [(api.SOLVER_FLAG_ALLOW_DOWNGRADE, 1)] - solver = api.Solver(self.pool, solver_options) - _specs = [convert_spec_to_conda_build(s) for s in specs] _constraints = [convert_spec_to_conda_build(s) for s in constraints or []] + if self._use_cache: + solver, pool = _get_solver_cached( + self.channels, self.platform, tuple(_constraints) + ) + else: + solver, pool = _get_solver( + self.channels, self.platform, tuple(_constraints) + ) + print_debug( "MAMBA running solver for specs \n\n%s\nconstraints: %s\n", pprint.pformat(_specs), pprint.pformat(_constraints), ) - for constraint in _constraints: - solver.add_pin(constraint) solver.add_jobs(_specs, api.SOLVER_INSTALL) success = solver.solve() @@ -143,10 +172,12 @@ def solve( print_warning( "MAMBA failed to solve specs \n\n%s\n\nwith " "constraints \n\n%s\n\nfor channels " + "\n\n%s\n\non platform " "\n\n%s\n\nThe reported errors are:\n\n%s\n", textwrap.indent(pprint.pformat(_specs), " "), textwrap.indent(pprint.pformat(_constraints), " "), textwrap.indent(pprint.pformat(self.channels), " "), + textwrap.indent(pprint.pformat(self.platform), " "), textwrap.indent(solver.explain_problems(), " "), ) err = solver.explain_problems() @@ -154,7 +185,7 @@ def solve( run_exports = copy.deepcopy(DEFAULT_RUN_EXPORTS) else: t = api.Transaction( - self.pool, + pool, solver, PACKAGE_CACHE, ) @@ -215,6 +246,5 @@ def _get_run_exports( return run_exports -@lru_cache(maxsize=128) def mamba_solver_factory(channels, platform): - return MambaSolver(list(channels), platform) + return MambaSolver(tuple(channels), platform, _use_cache=True) diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index 8d97cd7..aaa45d2 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -133,10 +133,12 @@ def solve( print_warning( "MAMBA failed to solve specs \n\n%s\n\nwith " "constraints \n\n%s\n\nfor channels " + "\n\n%s\n\non platform " "\n\n%s\n\nThe reported errors are:\n\n%s\n", textwrap.indent(pprint.pformat(specs), " "), textwrap.indent(pprint.pformat(constraints), " "), textwrap.indent(pprint.pformat(self.channels), " "), + textwrap.indent(pprint.pformat(self.platform_arch), " "), textwrap.indent(err, " "), ) success = False diff --git a/conda_forge_feedstock_check_solvable/utils.py b/conda_forge_feedstock_check_solvable/utils.py index f07a75c..f1803a0 100644 --- a/conda_forge_feedstock_check_solvable/utils.py +++ b/conda_forge_feedstock_check_solvable/utils.py @@ -149,7 +149,7 @@ def override_env_var(name, value): @contextlib.contextmanager def suppress_output(): - if "CONDA_FORGE_FEEDSTOCK_CHECK_SOLVABLE_DEBUG" in os.environ: + if "CONDA_FORGE_FEEDSTOCK_CHECK_SOLVABLE_DEBUG" in os.environ or VERBOSITY > 2: suppress = False else: suppress = True diff --git a/tests/test_solvers.py b/tests/test_solvers.py index 5609e77..a1b1962 100644 --- a/tests/test_solvers.py +++ b/tests/test_solvers.py @@ -1,7 +1,18 @@ +import inspect import pprint +import pytest from flaky import flaky +from conda_forge_feedstock_check_solvable.mamba_solver import ( + MambaSolver, + _get_solver_cached, + mamba_solver_factory, +) +from conda_forge_feedstock_check_solvable.rattler_solver import ( + RattlerSolver, + rattler_solver_factory, +) from conda_forge_feedstock_check_solvable.utils import apply_pins, suppress_output from conda_forge_feedstock_check_solvable.virtual_packages import ( virtual_package_repodata, @@ -215,3 +226,145 @@ def test_solvers_hang(solver_factory): ], ) assert res[0] + + +@pytest.mark.parametrize("mamba_factory", [MambaSolver, mamba_solver_factory]) +@pytest.mark.parametrize("rattler_factory", [RattlerSolver, rattler_solver_factory]) +def test_solvers_compare_output(mamba_factory, rattler_factory): + specs_linux = ( + "libutf8proc >=2.8.0,<3.0a0", + "orc >=2.0.1,<2.0.2.0a0", + "glog >=0.7.0,<0.8.0a0", + "libabseil * cxx17*", + "libgcc-ng >=12", + "libbrotlidec >=1.1.0,<1.2.0a0", + "bzip2 >=1.0.8,<2.0a0", + "libbrotlienc >=1.1.0,<1.2.0a0", + "libgoogle-cloud-storage >=2.24.0,<2.25.0a0", + "libstdcxx-ng >=12", + "re2", + "gflags >=2.2.2,<2.3.0a0", + "libabseil >=20240116.2,<20240117.0a0", + "libre2-11 >=2023.9.1,<2024.0a0", + "libgoogle-cloud >=2.24.0,<2.25.0a0", + "lz4-c >=1.9.3,<1.10.0a0", + "libbrotlicommon >=1.1.0,<1.2.0a0", + "aws-sdk-cpp >=1.11.329,<1.11.330.0a0", + "snappy >=1.2.0,<1.3.0a0", + "zstd >=1.5.6,<1.6.0a0", + "aws-crt-cpp >=0.26.9,<0.26.10.0a0", + "libzlib >=1.2.13,<2.0a0", + ) + constraints_linux = ("apache-arrow-proc * cpu", "arrow-cpp <0.0a0") + + specs_linux_again = ( + "glog >=0.7.0,<0.8.0a0", + "bzip2 >=1.0.8,<2.0a0", + "lz4-c >=1.9.3,<1.10.0a0", + "libbrotlidec >=1.1.0,<1.2.0a0", + "zstd >=1.5.6,<1.6.0a0", + "gflags >=2.2.2,<2.3.0a0", + "libzlib >=1.2.13,<2.0a0", + "libbrotlienc >=1.1.0,<1.2.0a0", + "re2", + "aws-sdk-cpp >=1.11.329,<1.11.330.0a0", + "libgoogle-cloud-storage >=2.24.0,<2.25.0a0", + "libgoogle-cloud >=2.24.0,<2.25.0a0", + "libstdcxx-ng >=12", + "libutf8proc >=2.8.0,<3.0a0", + "libabseil * cxx17*", + "snappy >=1.2.0,<1.3.0a0", + "__glibc >=2.17,<3.0.a0", + "orc >=2.0.1,<2.0.2.0a0", + "libgcc-ng >=12", + "libabseil >=20240116.2,<20240117.0a0", + "libbrotlicommon >=1.1.0,<1.2.0a0", + "libre2-11 >=2023.9.1,<2024.0a0", + "aws-crt-cpp >=0.26.9,<0.26.10.0a0", + ) + constraints_linux_again = ("arrow-cpp <0.0a0", "apache-arrow-proc * cuda") + + specs_win = ( + "re2", + "libabseil * cxx17*", + "vc >=14.2,<15", + "libbrotlidec >=1.1.0,<1.2.0a0", + "lz4-c >=1.9.3,<1.10.0a0", + "aws-sdk-cpp >=1.11.329,<1.11.330.0a0", + "libbrotlicommon >=1.1.0,<1.2.0a0", + "snappy >=1.2.0,<1.3.0a0", + "ucrt >=10.0.20348.0", + "orc >=2.0.1,<2.0.2.0a0", + "zstd >=1.5.6,<1.6.0a0", + "libcrc32c >=1.1.2,<1.2.0a0", + "libre2-11 >=2023.9.1,<2024.0a0", + "libbrotlienc >=1.1.0,<1.2.0a0", + "libcurl >=8.8.0,<9.0a0", + "libabseil >=20240116.2,<20240117.0a0", + "bzip2 >=1.0.8,<2.0a0", + "libgoogle-cloud >=2.24.0,<2.25.0a0", + "vc14_runtime >=14.29.30139", + "libzlib >=1.2.13,<2.0a0", + "libgoogle-cloud-storage >=2.24.0,<2.25.0a0", + "libutf8proc >=2.8.0,<3.0a0", + "aws-crt-cpp >=0.26.9,<0.26.10.0a0", + ) + constraints_win = ("arrow-cpp <0.0a0", "apache-arrow-proc * cuda") + + channels = (virtual_package_repodata(), "conda-forge", "msys2") + + platform = "linux-64" + mamba_solver = mamba_factory(channels, platform) + rattler_solver = rattler_factory(channels, platform) + mamba_solvable, mamba_err, mamba_solution = mamba_solver.solve( + specs_linux, constraints=constraints_linux + ) + rattler_solvable, rattler_err, rattler_solution = rattler_solver.solve( + specs_linux, constraints=constraints_linux + ) + assert set(mamba_solution or []) == set(rattler_solution or []) + assert mamba_solvable == rattler_solvable + + platform = "linux-64" + mamba_solver = mamba_factory(channels, platform) + rattler_solver = rattler_factory(channels, platform) + mamba_solvable, mamba_err, mamba_solution = mamba_solver.solve( + specs_linux_again, constraints=constraints_linux_again + ) + rattler_solvable, rattler_err, rattler_solution = rattler_solver.solve( + specs_linux_again, constraints=constraints_linux_again + ) + assert set(mamba_solution or []) == set(rattler_solution or []) + assert mamba_solvable == rattler_solvable + + platform = "linux-64" + mamba_solver = mamba_factory(channels, platform) + rattler_solver = rattler_factory(channels, platform) + mamba_solvable, mamba_err, mamba_solution = mamba_solver.solve( + specs_linux, constraints=constraints_linux + ) + rattler_solvable, rattler_err, rattler_solution = rattler_solver.solve( + specs_linux, constraints=constraints_linux + ) + assert set(mamba_solution or []) == set(rattler_solution or []) + assert mamba_solvable == rattler_solvable + + platform = "win-64" + mamba_solver = mamba_factory(channels, platform) + rattler_solver = rattler_factory(channels, platform) + mamba_solvable, mamba_err, mamba_solution = mamba_solver.solve( + specs_win, constraints=constraints_win + ) + rattler_solvable, rattler_err, rattler_solution = rattler_solver.solve( + specs_win, constraints=constraints_win + ) + assert set(mamba_solution or []) == set(rattler_solution or []) + assert mamba_solvable == rattler_solvable + + if inspect.isfunction(mamba_factory): + assert ( + _get_solver_cached.cache_info().misses == 3 + ), _get_solver_cached.cache_info() + + if hasattr(rattler_factory, "cache_info"): + assert rattler_factory.cache_info().misses == 2, rattler_factory.cache_info() From ed9fd2f2f4d7b9812394882a7be5a156fc4e26ca Mon Sep 17 00:00:00 2001 From: beckermr Date: Wed, 5 Jun 2024 07:38:06 -0500 Subject: [PATCH 27/35] TST fix tests for global caching --- .../mamba_solver.py | 5 +++++ tests/test_solvers.py | 17 ++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index 9130e11..eb4a7d1 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -248,3 +248,8 @@ def _get_run_exports( def mamba_solver_factory(channels, platform): return MambaSolver(tuple(channels), platform, _use_cache=True) + + +mamba_solver_factory.cache_info = _get_solver_cached.cache_info +mamba_solver_factory.cache_clear = _get_solver_cached.cache_clear +mamba_solver_factory.cache_parameters = _get_solver_cached.cache_parameters diff --git a/tests/test_solvers.py b/tests/test_solvers.py index a1b1962..6d8e7a2 100644 --- a/tests/test_solvers.py +++ b/tests/test_solvers.py @@ -6,7 +6,6 @@ from conda_forge_feedstock_check_solvable.mamba_solver import ( MambaSolver, - _get_solver_cached, mamba_solver_factory, ) from conda_forge_feedstock_check_solvable.rattler_solver import ( @@ -231,6 +230,10 @@ def test_solvers_hang(solver_factory): @pytest.mark.parametrize("mamba_factory", [MambaSolver, mamba_solver_factory]) @pytest.mark.parametrize("rattler_factory", [RattlerSolver, rattler_solver_factory]) def test_solvers_compare_output(mamba_factory, rattler_factory): + if inspect.isfunction(mamba_factory) and inspect.isfunction(rattler_factory): + mamba_factory.cache_clear() + rattler_factory.cache_clear() + specs_linux = ( "libutf8proc >=2.8.0,<3.0a0", "orc >=2.0.1,<2.0.2.0a0", @@ -361,10 +364,10 @@ def test_solvers_compare_output(mamba_factory, rattler_factory): assert set(mamba_solution or []) == set(rattler_solution or []) assert mamba_solvable == rattler_solvable - if inspect.isfunction(mamba_factory): + if inspect.isfunction(mamba_factory) and inspect.isfunction(rattler_factory): assert ( - _get_solver_cached.cache_info().misses == 3 - ), _get_solver_cached.cache_info() - - if hasattr(rattler_factory, "cache_info"): - assert rattler_factory.cache_info().misses == 2, rattler_factory.cache_info() + mamba_factory.cache_info().misses > rattler_factory.cache_info().misses + ), { + "mamba cache info": mamba_factory.cache_info(), + "rattler cache info": rattler_factory.cache_info(), + } From 554753507ee3181b93e5f2a5a99a60e59e803b36 Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 6 Jun 2024 06:00:24 -0500 Subject: [PATCH 28/35] REF cache subdir data only for mamba --- .../check_solvable.py | 50 ++++++++++----- .../mamba_solver.py | 35 ++++------- .../mamba_utils.py | 61 ++++++++----------- tests/test_solvers.py | 18 ++++++ tests/test_virtual_packages.py | 5 +- 5 files changed, 93 insertions(+), 76 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 48ab8e8..03251df 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -33,7 +33,8 @@ def is_recipe_solvable( timeout=600, build_platform=None, verbosity=1, - solver="mamba", + solver="rattler", + fail_fast=False, ) -> Tuple[bool, List[str], Dict[str, bool]]: """Compute if a recipe is solvable. @@ -60,6 +61,9 @@ def is_recipe_solvable( (gobbs of output). solver : str The solver to use. One of `mamba` or `rattler`. + fail_fast : bool + If True, then the function will return as soon as it finds a non-solvable + configuration. Returns ------- @@ -79,6 +83,7 @@ def is_recipe_solvable( verbosity=verbosity, solver=solver, timeout_timer=TimeoutTimer(timeout if timeout is not None else 6e5), + fail_fast=fail_fast, ) except TimeoutTimerException: print_warning("SOLVER TIMEOUT for %s", feedstock_dir) @@ -98,6 +103,7 @@ def _is_recipe_solvable( verbosity=1, solver="mamba", timeout_timer=None, + fail_fast=False, ) -> Tuple[bool, List[str], Dict[str, bool]]: conda_forge_feedstock_check_solvable.utils.VERBOSITY = verbosity timeout_timer = timeout_timer or TimeoutTimer(6e5) @@ -161,12 +167,16 @@ def _is_recipe_solvable( additional_channels=additional_channels, solver_backend=solver, timeout_timer=timeout_timer, + fail_fast=fail_fast, ) solvable = solvable and _solvable cbc_name = os.path.basename(cbc_fname).rsplit(".", maxsplit=1)[0] errors.extend([f"{cbc_name}: {e}" for e in _errors]) solvable_by_cbc[cbc_name] = _solvable + if not solvable and fail_fast: + break + return solvable, errors, solvable_by_cbc @@ -179,6 +189,7 @@ def _is_recipe_solvable_on_platform( additional_channels=(), solver_backend="mamba", timeout_timer=None, + fail_fast=False, ): timeout_timer = timeout_timer or TimeoutTimer(6e5) @@ -265,22 +276,21 @@ def _is_recipe_solvable_on_platform( # now we loop through each one and check if we can solve it # we check run and host and ignore the rest print_debug("getting solver") - with suppress_output(): - if solver_backend == "rattler": - solver_factory = rattler_solver_factory - elif solver_backend == "mamba": - solver_factory = mamba_solver_factory - else: - raise ValueError(f"Unknown solver backend {solver_backend}") - - solver = solver_factory(tuple(channel_sources), f"{platform}-{arch}") - timeout_timer.raise_for_timeout() + if solver_backend == "rattler": + solver_factory = rattler_solver_factory + elif solver_backend == "mamba": + solver_factory = mamba_solver_factory + else: + raise ValueError(f"Unknown solver backend {solver_backend}") - build_solver = solver_factory( - tuple(channel_sources), - f"{build_platform}-{build_arch}", - ) - timeout_timer.raise_for_timeout() + solver = solver_factory(tuple(channel_sources), f"{platform}-{arch}") + timeout_timer.raise_for_timeout() + + build_solver = solver_factory( + tuple(channel_sources), + f"{build_platform}-{build_arch}", + ) + timeout_timer.raise_for_timeout() solvable = True errors = [] @@ -314,6 +324,8 @@ def _is_recipe_solvable_on_platform( solvable = solvable and _solvable if _err is not None: errors.append(_err) + if not solvable and fail_fast: + break run_constrained = list(set(run_constrained) | build_rx["strong_constrains"]) @@ -351,6 +363,8 @@ def _is_recipe_solvable_on_platform( solvable = solvable and _solvable if _err is not None: errors.append(_err) + if not solvable and fail_fast: + break if m.is_cross: if m.noarch or m.noarch_python: @@ -382,6 +396,8 @@ def _is_recipe_solvable_on_platform( solvable = solvable and _solvable if _err is not None: errors.append(_err) + if not solvable and fail_fast: + break tst_req = ( m.get_value("test/requires", []) @@ -402,6 +418,8 @@ def _is_recipe_solvable_on_platform( solvable = solvable and _solvable if _err is not None: errors.append(_err) + if not solvable and fail_fast: + break print_info("RUN EXPORT CACHE STATUS: %s", get_run_exports.cache_info()) print_info("SOLVER CACHE STATUS: %s", solver_factory.cache_info()) diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index eb4a7d1..b29ad1a 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -13,7 +13,6 @@ import copy import pprint import textwrap -from functools import lru_cache from typing import List, Tuple import libmambapy as api @@ -21,7 +20,10 @@ from conda.base.context import context from conda.models.match_spec import MatchSpec -from conda_forge_feedstock_check_solvable.mamba_utils import load_channels +from conda_forge_feedstock_check_solvable.mamba_utils import ( + get_cached_index, + load_channels, +) from conda_forge_feedstock_check_solvable.utils import ( DEFAULT_RUN_EXPORTS, convert_spec_to_conda_build, @@ -42,7 +44,7 @@ api.Context().channel_priority = api.ChannelPriority.kStrict -def _get_pool(channels, platform, constraints): +def _get_pool(channels, platform): with suppress_output(): pool = api.Pool() @@ -62,7 +64,7 @@ def _get_pool(channels, platform, constraints): def _get_solver(channels, platform, constraints): - pool = _get_pool(channels, platform, constraints) + pool = _get_pool(channels, platform) solver_options = [(api.SOLVER_FLAG_ALLOW_DOWNGRADE, 1)] solver = api.Solver(pool, solver_options) @@ -73,11 +75,6 @@ def _get_solver(channels, platform, constraints): return solver, pool -@lru_cache(maxsize=128) -def _get_solver_cached(channels, platform, constraints): - return _get_solver(channels, platform, constraints) - - class MambaSolver: """Run the mamba solver. @@ -94,10 +91,9 @@ class MambaSolver: >>> solver.solve(["xtensor 0.18"]) """ - def __init__(self, channels, platform, _use_cache=False): + def __init__(self, channels, platform): self.channels = channels self.platform = platform - self._use_cache = _use_cache def solve( self, @@ -149,14 +145,7 @@ def solve( _specs = [convert_spec_to_conda_build(s) for s in specs] _constraints = [convert_spec_to_conda_build(s) for s in constraints or []] - if self._use_cache: - solver, pool = _get_solver_cached( - self.channels, self.platform, tuple(_constraints) - ) - else: - solver, pool = _get_solver( - self.channels, self.platform, tuple(_constraints) - ) + solver, pool = _get_solver(self.channels, self.platform, tuple(_constraints)) print_debug( "MAMBA running solver for specs \n\n%s\nconstraints: %s\n", @@ -247,9 +236,9 @@ def _get_run_exports( def mamba_solver_factory(channels, platform): - return MambaSolver(tuple(channels), platform, _use_cache=True) + return MambaSolver(tuple(channels), platform) -mamba_solver_factory.cache_info = _get_solver_cached.cache_info -mamba_solver_factory.cache_clear = _get_solver_cached.cache_clear -mamba_solver_factory.cache_parameters = _get_solver_cached.cache_parameters +mamba_solver_factory.cache_info = get_cached_index.cache_info +mamba_solver_factory.cache_clear = get_cached_index.cache_clear +mamba_solver_factory.cache_parameters = get_cached_index.cache_parameters diff --git a/conda_forge_feedstock_check_solvable/mamba_utils.py b/conda_forge_feedstock_check_solvable/mamba_utils.py index d6b67cc..aad2336 100644 --- a/conda_forge_feedstock_check_solvable/mamba_utils.py +++ b/conda_forge_feedstock_check_solvable/mamba_utils.py @@ -2,43 +2,36 @@ # SPDX-License-Identifier: BSD-3-Clause # Copied from mamba 1.5.2 +import copy import urllib.parse from collections import OrderedDict +from functools import lru_cache import libmambapy as api from conda.base.constants import ChannelPriority from conda.base.context import context -from conda.core.index import check_allowlist from conda.gateways.connection.session import CondaHttpAuth -def get_index( - channel_urls=(), - prepend=True, +@lru_cache(maxsize=128) +def get_cached_index( + channel_url, platform=None, - use_local=False, - use_cache=False, - unknown=None, - prefix=None, repodata_fn="repodata.json", ): if isinstance(platform, str): platform = [platform, "noarch"] all_channels = [] - if use_local: - all_channels.append("local") - all_channels.extend(channel_urls) - if prepend: - all_channels.extend(context.channels) - check_allowlist(all_channels) + all_channels.append(channel_url) # Remove duplicates but retain order all_channels = list(OrderedDict.fromkeys(all_channels)) + orig_all_channels = copy.deepcopy(all_channels) dlist = api.DownloadTargetList() - index = [] + subdirs = [] def fixup_channel_spec(spec): at_count = spec.count("@") @@ -57,7 +50,9 @@ def fixup_channel_spec(spec): pkgs_dirs = api.MultiPackageCache(context.pkgs_dirs) api.create_cache_dir(str(pkgs_dirs.first_writable_path)) - for channel in api.get_channels(all_channels): + for orig_channel_name, channel in zip( + orig_all_channels, api.get_channels(all_channels) + ): for channel_platform, url in channel.platform_urls(with_credentials=True): full_url = CondaHttpAuth.add_binstar_token(url) @@ -66,29 +61,29 @@ def fixup_channel_spec(spec): ) needs_finalising = sd.download_and_check_targets(dlist) - index.append( + if needs_finalising: + sd.finalize_checks() + + subdirs.append( ( sd, { "platform": channel_platform, "url": url, "channel": channel, - "needs_finalising": needs_finalising, + "needs_finalising": False, + "input_channel": orig_channel_name, }, ) ) - - for sd, info in index: - if info["needs_finalising"]: - sd.finalize_checks() - dlist.add(sd) + dlist.add(sd) is_downloaded = dlist.download(api.MAMBA_DOWNLOAD_FAILFAST) if not is_downloaded: raise RuntimeError("Error downloading repodata.") - return index + return subdirs def load_channels( @@ -96,20 +91,16 @@ def load_channels( channels, repos, has_priority=None, - prepend=True, platform=None, - use_local=False, - use_cache=True, repodata_fn="repodata.json", ): - index = get_index( - channel_urls=channels, - prepend=prepend, - platform=platform, - use_local=use_local, - repodata_fn=repodata_fn, - use_cache=use_cache, - ) + index = [] + for channel in channels: + index += get_cached_index( + channel_url=channel, + platform=platform, + repodata_fn=repodata_fn, + ) if has_priority is None: has_priority = context.channel_priority in [ diff --git a/tests/test_solvers.py b/tests/test_solvers.py index 6d8e7a2..9801416 100644 --- a/tests/test_solvers.py +++ b/tests/test_solvers.py @@ -371,3 +371,21 @@ def test_solvers_compare_output(mamba_factory, rattler_factory): "mamba cache info": mamba_factory.cache_info(), "rattler cache info": rattler_factory.cache_info(), } + + +@pytest.mark.parametrize("mamba_factory", [MambaSolver, mamba_solver_factory]) +@pytest.mark.parametrize("rattler_factory", [RattlerSolver, rattler_solver_factory]) +def test_solvers_python(mamba_factory, rattler_factory): + channels = (virtual_package_repodata(), "conda-forge", "defaults", "msys2") + platform = "linux-64" + for _ in range(4): + mamba_solver = mamba_factory(channels, platform) + rattler_solver = rattler_factory(channels, platform) + mamba_solvable, mamba_err, mamba_solution = mamba_solver.solve( + ["python"], + ) + rattler_solvable, rattler_err, rattler_solution = rattler_solver.solve( + ["python"], + ) + assert set(mamba_solution or []) == set(rattler_solution or []) + assert mamba_solvable == rattler_solvable diff --git a/tests/test_virtual_packages.py b/tests/test_virtual_packages.py index 099c753..1841ff4 100644 --- a/tests/test_virtual_packages.py +++ b/tests/test_virtual_packages.py @@ -11,11 +11,11 @@ @flaky -def test_virtual_package(feedstock_dir, tmp_path_factory): +def test_virtual_package(feedstock_dir, tmp_path, solver): recipe_file = os.path.join(feedstock_dir, "recipe", "meta.yaml") os.makedirs(os.path.dirname(recipe_file), exist_ok=True) - with FakeRepoData(tmp_path_factory.mktemp("channel")) as repodata: + with FakeRepoData(tmp_path) as repodata: for pkg in [ FakePackage("fakehostvirtualpkgdep", depends=frozenset(["__virtual >=10"])), FakePackage("__virtual", version="10"), @@ -50,5 +50,6 @@ def test_virtual_package(feedstock_dir, tmp_path_factory): solvable, err, solve_by_variant = is_recipe_solvable( feedstock_dir, additional_channels=[repodata.channel_url], + solver=solver, ) assert solvable From 7c81bfbd2b7e233d4714afefe957ed91791c626d Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 6 Jun 2024 06:02:40 -0500 Subject: [PATCH 29/35] TST fail fast --- tests/test_check_solvable.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/test_check_solvable.py b/tests/test_check_solvable.py index d91f21c..d7b9653 100644 --- a/tests/test_check_solvable.py +++ b/tests/test_check_solvable.py @@ -62,7 +62,11 @@ def test_is_recipe_solvable_ok(feedstock_dir, solver): """, ) assert is_recipe_solvable( - feedstock_dir, solver=solver, verbosity=VERB, timeout=None + feedstock_dir, + solver=solver, + verbosity=VERB, + timeout=None, + fail_fast=True, )[0] @@ -140,6 +144,7 @@ def test_r_base_cross_solvable(solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) assert solvable, pprint.pformat(errors) @@ -152,6 +157,7 @@ def test_xgboost_solvable(solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) assert solvable, pprint.pformat(errors) @@ -164,6 +170,7 @@ def test_pandas_solvable(solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) assert solvable, pprint.pformat(errors) @@ -188,6 +195,7 @@ def test_arrow_solvable(tmp_path, solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -206,6 +214,7 @@ def test_guiqwt_solvable(tmp_path, solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -224,6 +233,7 @@ def test_datalad_solvable(tmp_path, solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -242,6 +252,7 @@ def test_grpcio_solvable(tmp_path, solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -265,6 +276,7 @@ def test_cupy_solvable(tmp_path, solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) @@ -321,6 +333,7 @@ def test_run_exports_constrains_conflict(feedstock_dir, tmp_path_factory, solver solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) assert solvable, pprint.pformat(errors) @@ -434,6 +447,7 @@ def test_arrow_solvable_timeout(tmp_path, solver): timeout=0.1, solver=solver, verbosity=VERB, + fail_fast=True, ) assert solvable assert errors == [] @@ -509,6 +523,7 @@ def test_pillow_solvable(tmp_path, solver): solver=solver, verbosity=VERB, timeout=None, + fail_fast=True, ) pprint.pprint(solvable_by_variant) assert solvable, pprint.pformat(errors) From a7ed1d09d3b1985039203902c5163b8b4d0fa172 Mon Sep 17 00:00:00 2001 From: "Matthew R. Becker" Date: Thu, 6 Jun 2024 08:26:56 -0500 Subject: [PATCH 30/35] Update conda_forge_feedstock_check_solvable/rattler_solver.py Co-authored-by: jaimergp --- conda_forge_feedstock_check_solvable/rattler_solver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index aaa45d2..85947f0 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -131,7 +131,7 @@ def solve( except Exception as e: err = str(e) print_warning( - "MAMBA failed to solve specs \n\n%s\n\nwith " + "RATTLER failed to solve specs \n\n%s\n\nwith " "constraints \n\n%s\n\nfor channels " "\n\n%s\n\non platform " "\n\n%s\n\nThe reported errors are:\n\n%s\n", From defd925eff361fe0464ff0dcc913fb6da3864faf Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 6 Jun 2024 08:40:31 -0500 Subject: [PATCH 31/35] BUG fix bugs in vpkg repodata --- conda_forge_feedstock_check_solvable/virtual_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/virtual_packages.py b/conda_forge_feedstock_check_solvable/virtual_packages.py index 9dce91b..7399703 100644 --- a/conda_forge_feedstock_check_solvable/virtual_packages.py +++ b/conda_forge_feedstock_check_solvable/virtual_packages.py @@ -64,7 +64,7 @@ def _write_subdir(self, subdir): out = { "info": {"subdir": subdir}, "packages": packages, - "paxkages.conda": {}, + "packages.conda": {}, "removed": [], "repodata_version": 1, } @@ -81,7 +81,7 @@ def _write_subdir(self, subdir): packages[fname] = info_dict (self.base_path / subdir).mkdir(exist_ok=True) - (self.base_path / subdir / "repodata.json").write_text(json.dumps(out)) + (self.base_path / subdir / "repodata.json").write_text(json.dumps(out, sort_keys=True)) def write(self): all_subdirs = ALL_PLATFORMS.copy() From 9b2de724794eb186dda879c401eba9820aadd24b Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 6 Jun 2024 08:43:20 -0500 Subject: [PATCH 32/35] BUG do not use msys2 on windows --- conda_forge_feedstock_check_solvable/check_solvable.py | 4 ++-- conda_forge_feedstock_check_solvable/rattler_solver.py | 3 ++- conda_forge_feedstock_check_solvable/virtual_packages.py | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/check_solvable.py b/conda_forge_feedstock_check_solvable/check_solvable.py index 03251df..3f8d528 100644 --- a/conda_forge_feedstock_check_solvable/check_solvable.py +++ b/conda_forge_feedstock_check_solvable/check_solvable.py @@ -207,9 +207,9 @@ def _is_recipe_solvable_on_platform( # channel_sources might be part of some zip_key channel_sources.extend([c.strip() for c in source.split(",")]) else: - channel_sources = ["conda-forge", "defaults", "msys2"] + channel_sources = ["conda-forge", "defaults"] - if "msys2" not in channel_sources: + if "msys2" not in channel_sources and platform.startswith("win"): channel_sources.append("msys2") if additional_channels: diff --git a/conda_forge_feedstock_check_solvable/rattler_solver.py b/conda_forge_feedstock_check_solvable/rattler_solver.py index 85947f0..e91bf0e 100644 --- a/conda_forge_feedstock_check_solvable/rattler_solver.py +++ b/conda_forge_feedstock_check_solvable/rattler_solver.py @@ -40,7 +40,8 @@ def __init__(self, channels, platform_arch) -> None: if c == "defaults": _channels.append("https://repo.anaconda.com/pkgs/main") _channels.append("https://repo.anaconda.com/pkgs/r") - _channels.append("https://repo.anaconda.com/pkgs/msys2") + if platform_arch.startswith("win"): + _channels.append("https://repo.anaconda.com/pkgs/msys2") else: _channels.append(c) self._channels = [Channel(c) for c in _channels] diff --git a/conda_forge_feedstock_check_solvable/virtual_packages.py b/conda_forge_feedstock_check_solvable/virtual_packages.py index 7399703..60887a8 100644 --- a/conda_forge_feedstock_check_solvable/virtual_packages.py +++ b/conda_forge_feedstock_check_solvable/virtual_packages.py @@ -81,7 +81,9 @@ def _write_subdir(self, subdir): packages[fname] = info_dict (self.base_path / subdir).mkdir(exist_ok=True) - (self.base_path / subdir / "repodata.json").write_text(json.dumps(out, sort_keys=True)) + (self.base_path / subdir / "repodata.json").write_text( + json.dumps(out, sort_keys=True) + ) def write(self): all_subdirs = ALL_PLATFORMS.copy() From 56ae20b804d78de6f7a18660479c15cce564fbf7 Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 6 Jun 2024 08:44:28 -0500 Subject: [PATCH 33/35] TST move mamba tests to another workflow --- .github/workflows/tests.yml | 5 --- .github/workflows/tests_mamba.yml | 69 +++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/tests_mamba.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6e0533e..b1cb9f7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,8 +67,3 @@ jobs: shell: bash -l {0} run: | pytest -vv --durations=0 --solver=rattler tests - - - name: test w/ mamba - shell: bash -l {0} - run: | - pytest -vv --durations=0 --solver=mamba tests diff --git a/.github/workflows/tests_mamba.yml b/.github/workflows/tests_mamba.yml new file mode 100644 index 0000000..9fc269a --- /dev/null +++ b/.github/workflows/tests_mamba.yml @@ -0,0 +1,69 @@ +name: tests-mamba + +on: + push: + branches: + - main + pull_request: null + +env: + PY_COLORS: "1" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + tests-mamba: + name: tests-mamba + runs-on: "ubuntu-latest" + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v3 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v2 + with: + channels: conda-forge,defaults + channel-priority: strict + show-channel-urls: true + miniforge-version: latest + miniforge-variant: Mambaforge + python-version: 3.11 + use-mamba: true + + - name: configure conda and install code + shell: bash -l {0} + run: | + mamba install --yes --file=requirements.txt + mamba install --yes pytest flaky pip python-build setuptools_scm>=7 setuptools>=45 toml + pip install -e . + + - name: test versions + shell: bash -el {0} + run: | + pip uninstall conda-forge-feedstock-check-solvable --yes + [[ $(python setup.py --version) != "0.0.0" ]] || exit 1 + + rm -rf dist/* + python setup.py sdist + pip install --no-deps --no-build-isolation dist/*.tar.gz + pushd .. + python -c "import conda_forge_feedstock_check_solvable; assert conda_forge_feedstock_check_solvable.__version__ != '0.0.0'" + popd + pip uninstall conda-forge-feedstock-check-solvable --yes + + rm -rf dist/* + python -m build --sdist . --outdir dist + pip install --no-deps --no-build-isolation dist/*.tar.gz + pushd .. + python -c "import conda_forge_feedstock_check_solvable; assert conda_forge_feedstock_check_solvable.__version__ != '0.0.0'" + popd + pip uninstall conda-forge-feedstock-check-solvable --yes + + python -m pip install -v --no-deps --no-build-isolation -e . + + - name: test w/ mamba + shell: bash -l {0} + run: | + pytest -vv --durations=0 --solver=mamba tests From abf35875c727e7430a7ef41caee2394a84ac6e9b Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 6 Jun 2024 10:19:20 -0500 Subject: [PATCH 34/35] BUG only use set_installed with constraints --- .../mamba_solver.py | 12 ++++++---- .../mamba_utils.py | 23 ++++++++----------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index b29ad1a..acdeba3 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -56,19 +56,21 @@ def _get_pool(channels, platform): platform=platform, has_priority=True, ) - for repo in repos: - # need set_installed for add_pin, not sure why - repo.set_installed() - return pool + return pool, repos def _get_solver(channels, platform, constraints): - pool = _get_pool(channels, platform) + pool, repos = _get_pool(channels, platform) solver_options = [(api.SOLVER_FLAG_ALLOW_DOWNGRADE, 1)] solver = api.Solver(pool, solver_options) + if constraints: + for repo in repos: + # need set_installed for add_pin, not sure why + repo.set_installed() + for constraint in constraints: solver.add_pin(constraint) diff --git a/conda_forge_feedstock_check_solvable/mamba_utils.py b/conda_forge_feedstock_check_solvable/mamba_utils.py index aad2336..e4b5531 100644 --- a/conda_forge_feedstock_check_solvable/mamba_utils.py +++ b/conda_forge_feedstock_check_solvable/mamba_utils.py @@ -3,6 +3,7 @@ # Copied from mamba 1.5.2 import copy +import os import urllib.parse from collections import OrderedDict from functools import lru_cache @@ -124,25 +125,21 @@ def load_channels( priority = channel_prio else: priority = 0 + if has_priority: - subpriority = 0 + # as done in conda-libmamba-solver + subpriority = 1 else: subpriority = subprio_index subprio_index -= 1 - if not subdir.loaded() and entry["platform"] != "noarch": - # ignore non-loaded subdir if channel is != noarch - continue + cache_path = str(subdir.cache_path()) + if os.path.exists(cache_path.replace(".json", ".solv")): + cache_path = cache_path.replace(".json", ".solv") - if context.verbosity != 0 and not context.json: - print( - "Channel: {}, platform: {}, prio: {} : {}".format( - entry["channel"], entry["platform"], priority, subpriority - ) - ) - print("Cache path: ", subdir.cache_path()) - - repo = subdir.create_repo(pool) + repo = api.Repo( + pool, entry["url"], cache_path, urllib.parse.quote(entry["url"]) + ) repo.set_priority(priority, subpriority) repos.append(repo) From 24b4f7ee1eaea35ef9d8f5650884a977637a5b90 Mon Sep 17 00:00:00 2001 From: beckermr Date: Thu, 6 Jun 2024 10:48:53 -0500 Subject: [PATCH 35/35] BUG only install one thing --- .../mamba_solver.py | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/conda_forge_feedstock_check_solvable/mamba_solver.py b/conda_forge_feedstock_check_solvable/mamba_solver.py index acdeba3..abf3de3 100644 --- a/conda_forge_feedstock_check_solvable/mamba_solver.py +++ b/conda_forge_feedstock_check_solvable/mamba_solver.py @@ -10,8 +10,12 @@ https://gist.github.com/wolfv/cd12bd4a448c77ff02368e97ffdf495a. """ +import atexit import copy +import os import pprint +import shutil +import tempfile import textwrap from typing import List, Tuple @@ -44,6 +48,26 @@ api.Context().channel_priority = api.ChannelPriority.kStrict +def _make_installed_repo(): + # tmp directory in github actions + runner_tmp = os.environ.get("RUNNER_TEMP") + tmp_dir = tempfile.mkdtemp(dir=runner_tmp) + + if not runner_tmp: + # no need to bother cleaning up on CI + def clean(): + shutil.rmtree(tmp_dir, ignore_errors=True) + + atexit.register(clean) + + pth = os.path.join(tmp_dir, "installed", "repodata.json") + os.makedirs(os.path.dirname(pth), exist_ok=True) + with open(pth, "w") as f: + f.write("{}") + + return pth + + def _get_pool(channels, platform): with suppress_output(): pool = api.Pool() @@ -67,12 +91,12 @@ def _get_solver(channels, platform, constraints): solver = api.Solver(pool, solver_options) if constraints: - for repo in repos: - # need set_installed for add_pin, not sure why - repo.set_installed() + # add_pin needs an "installed" Repo to store the pin info + repo = api.Repo(pool, "installed", _make_installed_repo(), "") + repo.set_installed() - for constraint in constraints: - solver.add_pin(constraint) + for constraint in constraints: + solver.add_pin(constraint) return solver, pool