From 95954397f542e87d9b2ebe5e9dbc538415b6af6f Mon Sep 17 00:00:00 2001 From: conda-bot <18747875+conda-bot@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:34:11 -0500 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=A4=96=20updated=20file(s)=20(#866)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 8a6656d71..0164674b1 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -80,7 +80,7 @@ jobs: - if: github.event.comment.body != '@conda-bot render' id: create # no-op if no commits were made - uses: peter-evans/create-pull-request@6cd32fd93684475c31847837f87bb135d40a2b79 # v7.0.3 + uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 # v7.0.1 with: push-to-fork: ${{ env.FORK }} token: ${{ secrets.SYNC_TOKEN }} From 621c25e20f76f6060a058a2eb4d21efa61f49963 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 08:49:57 -0700 Subject: [PATCH 2/5] Bump peter-evans/create-pull-request in /.github/workflows (#867) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 7.0.1 to 7.0.5. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20...5e914681df9dc83aa4e4905692ca88beb2f9e91f) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 0164674b1..69a65aee6 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -80,7 +80,7 @@ jobs: - if: github.event.comment.body != '@conda-bot render' id: create # no-op if no commits were made - uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 # v7.0.1 + uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5 with: push-to-fork: ${{ env.FORK }} token: ${{ secrets.SYNC_TOKEN }} From 20b219a356935940e6c9ccd9b0d1bfecaa3f0e4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 08:07:27 -0700 Subject: [PATCH 3/5] Bump actions/checkout from 4.1.7 to 4.2.0 in /.github/workflows (#869) Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.7 to 4.2.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/692973e3d937129bcbf40652eb9f2f61becf3332...d632683dd7b4114ad314bca15554477dd762a938) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs.yml | 2 +- .github/workflows/labels.yml | 2 +- .github/workflows/main.yml | 6 +++--- .github/workflows/update.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 253d1649a..ff7b53e34 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,7 @@ jobs: run: shell: bash -el {0} steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 with: diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 6fd09350b..543a4dca6 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -23,7 +23,7 @@ jobs: GLOBAL: https://raw.githubusercontent.com/conda/infra/main/.github/global.yml LOCAL: .github/labels.yml steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - id: has_local uses: andstor/file-existence-action@076e0072799f4942c8bc574a82233e1e4d13e9d6 # v3.0.0 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d2be7c2ca..b1e3a60f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,7 +96,7 @@ jobs: PYTHONUNBUFFERED: "1" steps: - name: Retrieve the source code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 with: fetch-depth: 0 - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4 @@ -179,7 +179,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Retrieve the source code - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 - name: Report failures uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2 env: @@ -220,7 +220,7 @@ jobs: steps: # Clean checkout of specific git ref needed for package metadata version # which needs env vars GIT_DESCRIBE_TAG and GIT_BUILD_STR: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 with: ref: ${{ github.ref }} clean: true diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 69a65aee6..91224cfec 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -44,7 +44,7 @@ jobs: echo REPOSITORY=$(curl --silent ${{ github.event.issue.pull_request.url }} | jq --raw-output '.head.repo.full_name') >> $GITHUB_ENV echo REF=$(curl --silent ${{ github.event.issue.pull_request.url }} | jq --raw-output '.head.ref') >> $GITHUB_ENV - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 with: repository: ${{ env.REPOSITORY || github.repository }} ref: ${{ env.REF || '' }} From 86b0514030d066b672d82981f4e1e884c28f6982 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 08:07:51 -0700 Subject: [PATCH 4/5] [pre-commit.ci] pre-commit autoupdate (#870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/python-jsonschema/check-jsonschema: 0.29.2 → 0.29.3](https://github.com/python-jsonschema/check-jsonschema/compare/0.29.2...0.29.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e99d9a62e..24e9873fc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,6 +36,6 @@ repos: - id: shellcheck exclude: ^(constructor/header.sh|constructor/osx/.*sh) - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.3 hooks: - id: check-github-workflows From d5efecbd675116ec29ca5bfc9d590044458b6d97 Mon Sep 17 00:00:00 2001 From: Marco Esters Date: Tue, 1 Oct 2024 10:56:34 -0700 Subject: [PATCH 5/5] Improve standalone executable type detection and handling (#864) * Replace string values for standalone binary with enum * Explicitly test for micromamba * Replace is_micromamba with enum comparison * Remove circular import * Convert Path objects into str in indentify_conda_exe * Add news file * Add guard for None-type exe_version * Add missing guard for None-type exe versions * Loosen requirements for mamba help text * Add guards for None-type versions for skips * Do not use Enum * Update constructor/utils.py Co-authored-by: jaimergp * Simplify logger warning logic * Skip comments in explicit env file tests --------- Co-authored-by: jaimergp --- constructor/main.py | 26 ++++++----- constructor/utils.py | 46 ++++++++++++-------- constructor/winexe.py | 5 ++- news/864-improve-standalone-binary-detection | 19 ++++++++ tests/test_examples.py | 27 +++++++++--- 5 files changed, 86 insertions(+), 37 deletions(-) create mode 100644 news/864-improve-standalone-binary-detection diff --git a/constructor/main.py b/constructor/main.py index 98e6a080c..2f724bafd 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -21,7 +21,7 @@ from .construct import parse as construct_parse from .construct import verify as construct_verify from .fcp import main as fcp_main -from .utils import identify_conda_exe, normalize_path, yield_lines +from .utils import StandaloneExe, identify_conda_exe, normalize_path, yield_lines DEFAULT_CACHE_DIR = os.getenv('CONSTRUCTOR_CACHE', '~/.conda/constructor') @@ -92,7 +92,13 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, if platform != cc_platform and 'pkg' in itypes and not cc_platform.startswith('osx-'): sys.exit("Error: cannot construct a macOS 'pkg' installer on '%s'" % cc_platform) - if osname == "win" and "micromamba" in os.path.basename(info['_conda_exe']): + + exe_type, exe_version = identify_conda_exe(info.get("_conda_exe")) + if exe_version is not None: + exe_version = Version(exe_version) + info["_conda_exe_type"] = exe_type + info["_conda_exe_version"] = exe_version + if osname == "win" and exe_type == StandaloneExe.MAMBA: # TODO: Investigate errors on Windows and re-enable sys.exit("Error: micromamba is not supported on Windows installers.") @@ -172,17 +178,13 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, else: env_config[config_key] = value - try: - exe_name, exe_version = identify_conda_exe(info.get("_conda_exe")) - except OSError as exc: + if exe_type is None or exe_version is None: logger.warning( - "Could not identify conda-standalone / micromamba version (%s). " - "Will assume it is compatible with shortcuts.", - exc, - ) - exe_name, exe_version = None, None - if sys.platform != "win32" and exe_name is not None and ( - exe_name == "micromamba" or Version(exe_version) < Version("23.11.0") + "Could not identify conda-standalone / micromamba version. " + "Will assume it is compatible with shortcuts." + ) + elif sys.platform != "win32" and ( + exe_type != StandaloneExe.CONDA or exe_version < Version("23.11.0") ): logger.warning("conda-standalone 23.11.0 or above is required for shortcuts on Unix.") info['_enable_shortcuts'] = "incompatible" diff --git a/constructor/utils.py b/constructor/utils.py index 7bdf1ec5c..dc44f838e 100644 --- a/constructor/utils.py +++ b/constructor/utils.py @@ -11,9 +11,11 @@ import sys from io import StringIO from os import environ, sep, unlink -from os.path import basename, isdir, isfile, islink, join, normpath +from os.path import isdir, isfile, islink, join, normpath +from pathlib import Path from shutil import rmtree -from subprocess import check_call, check_output +from subprocess import CalledProcessError, check_call, check_output +from typing import Tuple, Union from ruamel.yaml import YAML @@ -23,6 +25,11 @@ yaml.indent(mapping=2, sequence=4, offset=2) +class StandaloneExe: + CONDA = "conda" + MAMBA = "mamba" + + def explained_check_call(args): """ Execute a system process and debug the invocation @@ -153,7 +160,7 @@ def ensure_transmuted_ext(info, url): """ if ( info.get("transmute_file_type") == ".conda" - and "micromamba" in basename(info.get("_conda_exe", "")) + and info.get("_conda_exe_type") == StandaloneExe.MAMBA ): if url.lower().endswith(".tar.bz2"): url = url[:-8] + ".conda" @@ -215,15 +222,13 @@ def yield_lines(path): yield line -def shortcuts_flags(info, conda_exe=None): +def shortcuts_flags(info) -> str: menu_packages = info.get("menu_packages") - conda_exe = conda_exe or info.get("_conda_exe", "") - is_micromamba = "micromamba" in basename(conda_exe).lower() if menu_packages is None: # not set: we create all shortcuts (default behaviour) return "" if menu_packages: - if is_micromamba: + if info.get("_conda_exe_type") == StandaloneExe.MAMBA: logger.warning( "Micromamba does not support '--shortcuts-only'. " "Will install all shortcuts." @@ -252,19 +257,24 @@ def approx_size_kb(info, which="pkgs"): return int(math.ceil(size_bytes/1000)) -def identify_conda_exe(conda_exe=None): +def identify_conda_exe(conda_exe: Union[str, Path] = None) -> Tuple[StandaloneExe, str]: if conda_exe is None: conda_exe = normalize_path(join(sys.prefix, "standalone_conda", "conda.exe")) - output = check_output([conda_exe, "--version"], text=True) - output = output.strip() - fields = output.split() - if "conda" in fields: - name = "conda-standalone" - version = fields[1] - else: - name = "micromamba" - version = output.strip() - return name, version + if isinstance(conda_exe, Path): + conda_exe = str(conda_exe) + try: + output_version = check_output([conda_exe, "--version"], text=True) + output_version = output_version.strip() + fields = output_version.split() + if "conda" in fields: + return StandaloneExe.CONDA, fields[1] + # micromamba only returns the version number + output_help = check_output([conda_exe, "--help"], text=True) + if "mamba" in output_help: + return StandaloneExe.MAMBA, output_version + except CalledProcessError as exc: + logger.warning(f"Could not identify standalone binary {exc}.") + return None, None def win_str_esc(s, newlines=True): diff --git a/constructor/winexe.py b/constructor/winexe.py index 8d6d7fd4c..16bcca7b5 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -172,6 +172,9 @@ def setup_envs_commands(info, dir_path): for env_name in info.get("_extra_envs_info", {}): lines += ["", ""] env_info = info["extra_envs"][env_name] + # Needed for shortcuts_flags function + if "_conda_exe_type" not in env_info: + env_info["_conda_exe_type"] = info.get("_conda_exe_type") channel_info = { "channels": env_info.get("channels", info.get("channels", ())), "channels_remap": env_info.get("channels_remap", info.get("channels_remap", ())) @@ -185,7 +188,7 @@ def setup_envs_commands(info, dir_path): conda_meta=join("$INSTDIR", "envs", env_name, "conda-meta"), history_abspath=join(dir_path, "envs", env_name, "conda-meta", "history"), channels=",".join(get_final_channels(channel_info)), - shortcuts=shortcuts_flags(env_info, conda_exe=info.get("_conda_exe")), + shortcuts=shortcuts_flags(env_info), register_envs=str(info.get("register_envs", True)).lower(), ).splitlines() diff --git a/news/864-improve-standalone-binary-detection b/news/864-improve-standalone-binary-detection new file mode 100644 index 000000000..761b0a3fa --- /dev/null +++ b/news/864-improve-standalone-binary-detection @@ -0,0 +1,19 @@ +### Enhancements + +* Improve detection and handling of standalone executable type. (#864) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_examples.py b/tests/test_examples.py index 380d68f6f..549cfd835 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -18,7 +18,7 @@ from conda.core.prefix_data import PrefixData from conda.models.version import VersionOrder as Version -from constructor.utils import identify_conda_exe +from constructor.utils import StandaloneExe, identify_conda_exe if sys.platform == "darwin": from constructor.osxpkg import calculate_install_dir @@ -36,6 +36,8 @@ ON_CI = os.environ.get("CI") CONSTRUCTOR_CONDA_EXE = os.environ.get("CONSTRUCTOR_CONDA_EXE") CONDA_EXE, CONDA_EXE_VERSION = identify_conda_exe(CONSTRUCTOR_CONDA_EXE) +if CONDA_EXE_VERSION is not None: + CONDA_EXE_VERSION = Version(CONDA_EXE_VERSION) CONSTRUCTOR_DEBUG = bool(os.environ.get("CONSTRUCTOR_DEBUG")) if artifacts_path := os.environ.get("CONSTRUCTOR_EXAMPLES_KEEP_ARTIFACTS"): KEEP_ARTIFACTS_PATH = Path(artifacts_path) @@ -355,8 +357,9 @@ def _example_path(example_name): return REPO_DIR / "examples" / example_name -def _is_micromamba(path): - return "micromamba" in Path(path).stem +def _is_micromamba(path) -> bool: + name, _ = identify_conda_exe(path) + return name == StandaloneExe.MAMBA def test_example_customize_controls(tmp_path, request): @@ -391,7 +394,11 @@ def test_example_extra_files(tmp_path, request): @pytest.mark.xfail( - CONDA_EXE == "conda-standalone" and Version(CONDA_EXE_VERSION) < Version("23.11.0a0"), + ( + CONDA_EXE == StandaloneExe.CONDA + and CONDA_EXE_VERSION is not None + and CONDA_EXE_VERSION < Version("23.11.0a0") + ), reason="Known issue with conda-standalone<=23.10: shortcuts are created but not removed.", ) def test_example_miniforge(tmp_path, request): @@ -565,7 +572,11 @@ def test_example_scripts(tmp_path, request): @pytest.mark.skipif( - CONDA_EXE == "micromamba" or Version(CONDA_EXE_VERSION) < Version("23.11.0a0"), + ( + CONDA_EXE == StandaloneExe.MAMBA + or CONDA_EXE_VERSION is None + or CONDA_EXE_VERSION < Version("23.11.0a0") + ), reason="menuinst v2 requires conda-standalone>=23.11.0; micromamba is not supported yet", ) def test_example_shortcuts(tmp_path, request): @@ -723,7 +734,11 @@ def test_example_from_explicit(tmp_path, request): [sys.executable, "-mconda", "list", "-p", install_dir, "--explicit", "--md5"], text=True, ) - assert out == (input_path / "explicit_linux-64.txt").read_text() + expected = (input_path / "explicit_linux-64.txt").read_text() + # Filter comments + out = [line for line in out.split("\n") if not line.startswith("#")] + expected = [line for line in expected.split("\n") if not line.startswith("#")] + assert out == expected def test_register_envs(tmp_path, request):