From cb67840c8d474cefedc875767661bff4b03c3bcf Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Mon, 11 Mar 2024 17:49:14 +0100 Subject: [PATCH 1/4] Defer `.index._apply_instructions` and `.conda_interface.get_index` deprecations (#5226) * Re-add .index._apply_instructions 24.5 deprecation Signed-off-by: Marcel Bargull * Defer .conda_interface.get_index deprecation 24.5 Signed-off-by: Marcel Bargull --------- Signed-off-by: Marcel Bargull --- conda_build/conda_interface.py | 2 ++ conda_build/index.py | 59 +++++++++++++++++++++++++++++++++- news/5203-remove-deprecations | 2 -- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/conda_build/conda_interface.py b/conda_build/conda_interface.py index 3f25e89591..5d5c455d07 100644 --- a/conda_build/conda_interface.py +++ b/conda_build/conda_interface.py @@ -58,11 +58,13 @@ walk_prefix, win_path_to_unix, ) +from conda.exports import get_index as _get_index from conda.gateways.disk.read import compute_sum from conda.models.channel import get_conda_build_local_url # noqa: F401 from .deprecations import deprecated +deprecated.constant("24.1.0", "24.5.0", "get_index", _get_index) # TODO: Go to references of all properties below and import them from `context` instead binstar_upload = context.binstar_upload default_python = context.default_python diff --git a/conda_build/index.py b/conda_build/index.py index c3968d238a..cd36cc9cac 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -14,7 +14,13 @@ from . import conda_interface, utils from .conda_interface import CondaHTTPError, context, url_path from .deprecations import deprecated -from .utils import JSONDecodeError, get_logger, on_win +from .utils import ( + CONDA_PACKAGE_EXTENSION_V1, + CONDA_PACKAGE_EXTENSION_V2, + JSONDecodeError, + get_logger, + on_win, +) log = get_logger(__name__) @@ -243,3 +249,54 @@ def _delegated_update_index( current_index_versions=current_index_versions, debug=debug, ) + + +@deprecated( + "24.1.0", "24.5.0", addendum="Use `conda_index._apply_instructions` instead." +) +def _apply_instructions(subdir, repodata, instructions): + repodata.setdefault("removed", []) + utils.merge_or_update_dict( + repodata.get("packages", {}), + instructions.get("packages", {}), + merge=False, + add_missing_keys=False, + ) + # we could have totally separate instructions for .conda than .tar.bz2, but it's easier if we assume + # that a similarly-named .tar.bz2 file is the same content as .conda, and shares fixes + new_pkg_fixes = { + k.replace(CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2): v + for k, v in instructions.get("packages", {}).items() + } + + utils.merge_or_update_dict( + repodata.get("packages.conda", {}), + new_pkg_fixes, + merge=False, + add_missing_keys=False, + ) + utils.merge_or_update_dict( + repodata.get("packages.conda", {}), + instructions.get("packages.conda", {}), + merge=False, + add_missing_keys=False, + ) + + for fn in instructions.get("revoke", ()): + for key in ("packages", "packages.conda"): + if fn.endswith(CONDA_PACKAGE_EXTENSION_V1) and key == "packages.conda": + fn = fn.replace(CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2) + if fn in repodata[key]: + repodata[key][fn]["revoked"] = True + repodata[key][fn]["depends"].append("package_has_been_revoked") + + for fn in instructions.get("remove", ()): + for key in ("packages", "packages.conda"): + if fn.endswith(CONDA_PACKAGE_EXTENSION_V1) and key == "packages.conda": + fn = fn.replace(CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2) + popped = repodata[key].pop(fn, None) + if popped: + repodata["removed"].append(fn) + repodata["removed"].sort() + + return repodata diff --git a/news/5203-remove-deprecations b/news/5203-remove-deprecations index fb77c3b149..5021c12907 100644 --- a/news/5203-remove-deprecations +++ b/news/5203-remove-deprecations @@ -30,7 +30,6 @@ * Remove `conda_build.conda_interface.display_actions`. (#5203) * Remove `conda_build.conda_interface.execute_actions`. (#5203) * Remove `conda_build.conda_interface.execute_plan`. (#5203) -* Remove `conda_build.conda_interface.get_index`. (#5203) * Remove `conda_build.conda_interface.install_actions`. (#5203) * Remove `conda_build.conda_interface.linked`. (#5203) * Remove `conda_build.conda_interface.linked_data`. (#5203) @@ -44,7 +43,6 @@ * Remove `conda_build.index.REPODATA_FROM_PKGS_JSON_FN`. (#5203) * Remove `conda_build.index.CHANNELDATA_FIELDS`. (#5203) * Remove `conda_build.index._clear_newline_chars`. (#5203) -* Remove `conda_build.index._apply_instructions`. (#5203) * Remove `conda_build.index._get_jinja2_environment`. (#5203) * Remove `conda_build.index._maybe_write`. (#5203) * Remove `conda_build.index._make_build_string`. (#5203) From 4bc2fd821aafb27561f96cb91e54eb5e83c27979 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Mon, 11 Mar 2024 22:19:02 +0100 Subject: [PATCH 2/4] Add compatibility for LIEF=0.14 (#5228) * Add compatibility for LIEF=0.14 Signed-off-by: Marcel Bargull * Add news/5228-lief-0.14-compat Signed-off-by: Marcel Bargull * TMP: Test with conda-forge/label/lief_rc::py-lief >=0.14 Signed-off-by: Marcel Bargull * Replace lief.Binary.name (lief<0.14) by str(file) Signed-off-by: Marcel Bargull * Revert "TMP: Test with conda-forge/label/lief_rc::py-lief >=0.14" This reverts commit cb5df71c9afd743262a9fd1476f6f9c3cc98b126. Signed-off-by: Marcel Bargull --------- Signed-off-by: Marcel Bargull --- conda_build/os_utils/liefldd.py | 72 +++++++++++++++++++-------------- news/5228-lief-0.14-compat | 19 +++++++++ 2 files changed, 60 insertions(+), 31 deletions(-) create mode 100644 news/5228-lief-0.14-compat diff --git a/conda_build/os_utils/liefldd.py b/conda_build/os_utils/liefldd.py index 9f358f619a..9b14454c4f 100644 --- a/conda_build/os_utils/liefldd.py +++ b/conda_build/os_utils/liefldd.py @@ -27,6 +27,16 @@ lief.logging.disable() have_lief = True + try: + PE_HEADER_CHARACTERISTICS = lief.PE.Header.CHARACTERISTICS + except AttributeError: + # Fallback for lief<0.14. + PE_HEADER_CHARACTERISTICS = lief.PE.HEADER_CHARACTERISTICS + try: + EXE_FORMATS = lief.Binary.FORMATS + except AttributeError: + # Fallback for lief<0.14. + EXE_FORMATS = lief.EXE_FORMATS except ImportError: have_lief = False @@ -78,15 +88,15 @@ def codefile_class( if not (binary := ensure_binary(path)): return None elif ( - binary.format == lief.EXE_FORMATS.PE - and lief.PE.HEADER_CHARACTERISTICS.DLL in binary.header.characteristics_list + binary.format == EXE_FORMATS.PE + and PE_HEADER_CHARACTERISTICS.DLL in binary.header.characteristics_list ): return DLLfile - elif binary.format == lief.EXE_FORMATS.PE: + elif binary.format == EXE_FORMATS.PE: return EXEfile - elif binary.format == lief.EXE_FORMATS.MACHO: + elif binary.format == EXE_FORMATS.MACHO: return machofile - elif binary.format == lief.EXE_FORMATS.ELF: + elif binary.format == EXE_FORMATS.ELF: return elffile else: return None @@ -105,7 +115,7 @@ def get_libraries(file): result = [] binary = ensure_binary(file) if binary: - if binary.format == lief.EXE_FORMATS.PE: + if binary.format == EXE_FORMATS.PE: result = binary.libraries else: result = [ @@ -113,7 +123,7 @@ def get_libraries(file): ] # LIEF returns LC_ID_DYLIB name @rpath/libbz2.dylib in binary.libraries. Strip that. binary_name = None - if binary.format == lief.EXE_FORMATS.MACHO: + if binary.format == EXE_FORMATS.MACHO: binary_name = [ command.name for command in binary.commands @@ -174,7 +184,7 @@ def get_rpathy_thing_raw_partial(file, elf_attribute, elf_dyn_tag): rpaths = [] if binary: binary_format = binary.format - if binary_format == lief.EXE_FORMATS.ELF: + if binary_format == EXE_FORMATS.ELF: binary_type = binary.type if ( binary_type == lief.ELF.ELF_CLASS.CLASS32 @@ -182,7 +192,7 @@ def get_rpathy_thing_raw_partial(file, elf_attribute, elf_dyn_tag): ): rpaths = _get_elf_rpathy_thing(binary, elf_attribute, elf_dyn_tag) elif ( - binary_format == lief.EXE_FORMATS.MACHO + binary_format == EXE_FORMATS.MACHO and binary.has_rpath and elf_dyn_tag == lief.ELF.DYNAMIC_TAGS.RPATH ): @@ -232,7 +242,7 @@ def set_rpath(old_matching, new_rpath, file): binary = ensure_binary(file) if not binary: return - if binary.format == lief.EXE_FORMATS.ELF and ( + if binary.format == EXE_FORMATS.ELF and ( binary.type == lief.ELF.ELF_CLASS.CLASS32 or binary.type == lief.ELF.ELF_CLASS.CLASS64 ): @@ -244,7 +254,7 @@ def set_rpath(old_matching, new_rpath, file): def get_rpaths(file, exe_dirname, envroot, windows_root=""): rpaths, rpaths_type, binary_format, binary_type = get_runpaths_or_rpaths_raw(file) - if binary_format == lief.EXE_FORMATS.PE: + if binary_format == EXE_FORMATS.PE: # To allow the unix-y rpath code to work we consider # exes as having rpaths of env + CONDA_WINDOWS_PATHS # and consider DLLs as having no rpaths. @@ -259,9 +269,9 @@ def get_rpaths(file, exe_dirname, envroot, windows_root=""): rpaths.append("/".join((windows_root, "System32", "downlevel"))) rpaths.append(windows_root) if envroot: - # and not lief.PE.HEADER_CHARACTERISTICS.DLL in binary.header.characteristics_list: + # and not .DLL in binary.header.characteristics_list: rpaths.extend(list(_get_path_dirs(envroot))) - elif binary_format == lief.EXE_FORMATS.MACHO: + elif binary_format == EXE_FORMATS.MACHO: rpaths = [rpath.rstrip("/") for rpath in rpaths] return [from_os_varnames(binary_format, binary_type, rpath) for rpath in rpaths] @@ -299,13 +309,13 @@ def _inspect_linkages_this(filename, sysroot="", arch="native"): def to_os_varnames(binary, input_): """Don't make these functions - they are methods to match the API for elffiles.""" - if binary.format == lief.EXE_FORMATS.MACHO: + if binary.format == EXE_FORMATS.MACHO: return ( input_.replace("$SELFDIR", "@loader_path") .replace("$EXEDIR", "@executable_path") .replace("$RPATH", "@rpath") ) - elif binary.format == lief.EXE_FORMATS.ELF: + elif binary.format == EXE_FORMATS.ELF: if binary.ehdr.sz_ptr == 8: libdir = "/lib64" else: @@ -315,19 +325,19 @@ def to_os_varnames(binary, input_): def from_os_varnames(binary_format, binary_type, input_): """Don't make these functions - they are methods to match the API for elffiles.""" - if binary_format == lief.EXE_FORMATS.MACHO: + if binary_format == EXE_FORMATS.MACHO: return ( input_.replace("@loader_path", "$SELFDIR") .replace("@executable_path", "$EXEDIR") .replace("@rpath", "$RPATH") ) - elif binary_format == lief.EXE_FORMATS.ELF: + elif binary_format == EXE_FORMATS.ELF: if binary_type == lief.ELF.ELF_CLASS.CLASS64: libdir = "/lib64" else: libdir = "/lib" return input_.replace("$ORIGIN", "$SELFDIR").replace("$LIB", libdir) - elif binary_format == lief.EXE_FORMATS.PE: + elif binary_format == EXE_FORMATS.PE: return input_ @@ -344,10 +354,10 @@ def _get_path_dirs(prefix): def get_uniqueness_key(file): binary = ensure_binary(file) if not binary: - return lief.EXE_FORMATS.UNKNOWN - elif binary.format == lief.EXE_FORMATS.MACHO: - return binary.name - elif binary.format == lief.EXE_FORMATS.ELF and ( # noqa + return EXE_FORMATS.UNKNOWN + elif binary.format == EXE_FORMATS.MACHO: + return str(file) + elif binary.format == EXE_FORMATS.ELF and ( # noqa binary.type == lief.ELF.ELF_CLASS.CLASS32 or binary.type == lief.ELF.ELF_CLASS.CLASS64 ): @@ -357,8 +367,8 @@ def get_uniqueness_key(file): ] if result: return result[0] - return binary.name - return binary.name + return str(file) + return str(file) def _get_resolved_location( @@ -467,7 +477,7 @@ def inspect_linkages_lief( default_paths = [] if not binary: default_paths = [] - elif binary.format == lief.EXE_FORMATS.ELF: + elif binary.format == EXE_FORMATS.ELF: if binary.type == lief.ELF.ELF_CLASS.CLASS64: default_paths = [ "$SYSROOT/lib64", @@ -477,9 +487,9 @@ def inspect_linkages_lief( ] else: default_paths = ["$SYSROOT/lib", "$SYSROOT/usr/lib"] - elif binary.format == lief.EXE_FORMATS.MACHO: + elif binary.format == EXE_FORMATS.MACHO: default_paths = ["$SYSROOT/usr/lib"] - elif binary.format == lief.EXE_FORMATS.PE: + elif binary.format == EXE_FORMATS.PE: # We do not include C:\Windows nor C:\Windows\System32 in this list. They are added in # get_rpaths() instead since we need to carefully control the order. default_paths = [ @@ -499,7 +509,7 @@ def inspect_linkages_lief( uniqueness_key = get_uniqueness_key(binary) if uniqueness_key not in already_seen: parent_exe_dirname = None - if binary.format == lief.EXE_FORMATS.PE: + if binary.format == EXE_FORMATS.PE: tmp_filename = filename2 while tmp_filename: if ( @@ -519,7 +529,7 @@ def inspect_linkages_lief( ) tmp_filename = filename2 rpaths_transitive = [] - if binary.format == lief.EXE_FORMATS.PE: + if binary.format == EXE_FORMATS.PE: rpaths_transitive = rpaths_by_binary[tmp_filename] else: while tmp_filename: @@ -534,7 +544,7 @@ def inspect_linkages_lief( "$RPATH/" + lib if not lib.startswith("/") and not lib.startswith("$") - and binary.format != lief.EXE_FORMATS.MACHO # noqa + and binary.format != EXE_FORMATS.MACHO # noqa else lib ) for lib in libraries @@ -556,7 +566,7 @@ def inspect_linkages_lief( # can be run case-sensitively if the user wishes. # """ - if binary.format == lief.EXE_FORMATS.PE: + if binary.format == EXE_FORMATS.PE: import random path_fixed = ( os.path.dirname(path_fixed) diff --git a/news/5228-lief-0.14-compat b/news/5228-lief-0.14-compat new file mode 100644 index 0000000000..602242c7d4 --- /dev/null +++ b/news/5228-lief-0.14-compat @@ -0,0 +1,19 @@ +### Enhancements + +* Add compatibility for LIEF=0.14. (#5227 via #5228) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* From d384ef09dfb59ae90c07021a71ea76ed26675eae Mon Sep 17 00:00:00 2001 From: conda-bot <18747875+conda-bot@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:02:17 -0500 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=94=84=20synced=20file(s)=20with=20co?= =?UTF-8?q?nda/infrastructure=20(#5230)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Conda Bot --- HOW_WE_USE_GITHUB.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HOW_WE_USE_GITHUB.md b/HOW_WE_USE_GITHUB.md index d0a4f4266f..46a13ecd98 100644 --- a/HOW_WE_USE_GITHUB.md +++ b/HOW_WE_USE_GITHUB.md @@ -225,7 +225,7 @@ This is a duplicate of [link to primary issue]; please feel free to conti 
 
 Please uninstall your current version of `conda` and reinstall the latest version.
-Feel free to use either the [miniconda](https://docs.conda.io/en/latest/miniconda.html)
+Feel free to use either the [miniconda](https://docs.anaconda.com/free/miniconda/)
 or [anaconda](https://www.anaconda.com/products/individual) installer,
 whichever is more appropriate for your needs.
From 17c51bdf3faeae9a8b609909f1acd1fef1594a1f Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 12 Mar 2024 08:35:14 -0500 Subject: [PATCH 4/4] Sync CI with conda (#5146) --- ci/github/.condarc => .github/condarc | 2 - .github/workflows/tests.yml | 359 +++++++++--------- pyproject.toml | 8 + ...ments-linux.txt => requirements-Linux.txt} | 0 ...s-windows.txt => requirements-Windows.txt} | 0 tests/requirements-ci.txt | 19 + ...ments-macos.txt => requirements-macOS.txt} | 0 tests/requirements.txt | 23 +- 8 files changed, 211 insertions(+), 200 deletions(-) rename ci/github/.condarc => .github/condarc (73%) rename tests/{requirements-linux.txt => requirements-Linux.txt} (100%) rename tests/{requirements-windows.txt => requirements-Windows.txt} (100%) create mode 100644 tests/requirements-ci.txt rename tests/{requirements-macos.txt => requirements-macOS.txt} (100%) diff --git a/ci/github/.condarc b/.github/condarc similarity index 73% rename from ci/github/.condarc rename to .github/condarc index 44a36fcc35..a76e773f8f 100644 --- a/ci/github/.condarc +++ b/.github/condarc @@ -2,7 +2,5 @@ auto_update_conda: False auto_activate_base: True notify_outdated_conda: False changeps1: False -pkgs_dirs: -- /usr/share/miniconda/envs/test/pkgs always_yes: True local_repodata_ttl: 7200 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c0b0e8ff59..090c389a6b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,3 @@ -# this is the sibling workflow to tests-skip.yml, it is required to work around -# the skipped but required checks issue: -# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks name: Tests on: @@ -32,20 +29,29 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true +env: + # https://conda.github.io/conda-libmamba-solver/user-guide/configuration/#advanced-options + CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED: true + jobs: # detect whether any code changes are included in this PR changes: runs-on: ubuntu-latest permissions: + # necessary to detect changes + # https://github.com/dorny/paths-filter#supported-workflows pull-requests: read outputs: code: ${{ steps.filter.outputs.code }} steps: - - uses: actions/checkout@v3 + - name: Checkout Source + uses: actions/checkout@v4 # dorny/paths-filter needs git clone for non-PR events - # https://github.com/marketplace/actions/paths-changes-filter#supported-workflows + # https://github.com/dorny/paths-filter#supported-workflows if: github.event_name != 'pull_request' - - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 + + - name: Filter Changes + uses: dorny/paths-filter@v3 id: filter with: filters: | @@ -65,6 +71,7 @@ jobs: runs-on: ubuntu-latest defaults: run: + # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell shell: bash -el {0} strategy: fail-fast: false @@ -89,75 +96,70 @@ jobs: conda-version: canary test-type: parallel env: - CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }} - CONDA_VERSION: ${{ contains('canary,release', matrix.conda-version) && 'conda' || format('conda={0}', matrix.conda-version) }} - REPLAY_NAME: Linux-${{ matrix.conda-version }}-Py${{ matrix.python-version }} - REPLAY_DIR: ${{ github.workspace }}/pytest-replay + CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev::' || '' }} + CONDA_VERSION: ${{ contains('canary|release', matrix.conda-version) && 'conda' || format('conda={0}', matrix.conda-version) }} PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }} - PYTEST_NUMPROCESSES: ${{ matrix.test-type == 'serial' && 0 || 'auto' }} steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout Source + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Timestamp - run: echo "TIMESTAMP=$(date -u "+%Y%m")" >> $GITHUB_ENV - shell: bash + - name: Hash + Timestamp + run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV - - name: Cache conda - uses: actions/cache@v3 + - name: Cache Conda + uses: actions/cache@v4 with: path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.TIMESTAMP }} + key: cache-${{ env.HASH }} - - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v2 + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3 with: - condarc-file: ./ci/github/.condarc - python-version: ${{ matrix.python-version }} + condarc-file: .github/condarc run-post: false # skip post cleanup - - name: Setup environment - run: | - conda install -q -y -c defaults \ - --file ./tests/requirements.txt \ - --file ./tests/requirements-linux.txt \ - ${{ env.CONDA_CHANNEL_LABEL }}::${{ env.CONDA_VERSION }} - pip install -e . --no-deps - - - name: Show info - run: | - conda info -a - conda list --show-channel-urls - - - name: Run tests - run: | - pytest \ - --color=yes \ - -v \ - -n "${{ env.PYTEST_NUMPROCESSES }}" \ - --basetemp "${{ runner.temp }}/${{ matrix.test-type }}" \ - --cov conda_build \ - --cov-append \ - --cov-branch \ - --cov-report xml \ - --replay-record-dir="${{ env.REPLAY_DIR }}" \ - --replay-base-name="${{ env.REPLAY_NAME }}" \ - -m "${{ env.PYTEST_MARKER }}" \ - ./tests - - - uses: codecov/codecov-action@v3 + - name: Conda Install + run: conda install + --yes + --file tests/requirements.txt + --file tests/requirements-${{ runner.os }}.txt + --file tests/requirements-ci.txt + python=${{ matrix.python-version }} + ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }} + + # TODO: how can we remove this step? + - name: Install Self + run: pip install -e . + + - name: Conda Info + run: conda info --verbose + + - name: Conda List + run: conda list --show-channel-urls + + - name: Run Tests + run: pytest + --cov=conda_build + -n auto + -m "${{ env.PYTEST_MARKER }}" + + - name: Upload Coverage + uses: codecov/codecov-action@v4 with: - flags: ${{ matrix.test-type }},${{ matrix.python-version }},linux-64 + flags: ${{ runner.os }},${{ runner.arch }},${{ matrix.python-version }},${{ matrix.test-type }} - - name: Upload Pytest Replay + - name: Upload Test Results if: '!cancelled()' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: ${{ env.REPLAY_NAME }}-${{ matrix.test-type }} - path: ${{ env.REPLAY_DIR }} + name: test-results-${{ env.HASH }} + path: | + .coverage + test-report.xml + retention-days: 1 # temporary, combined in aggregate below # windows test suite windows: @@ -181,77 +183,77 @@ jobs: conda-version: canary test-type: parallel env: + ErrorActionPreference: Stop # powershell exit on first error CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }} - REPLAY_NAME: Win-${{ matrix.conda-version }}-Py${{ matrix.python-version }} - REPLAY_DIR: ${{ github.workspace }}\pytest-replay PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial and not slow' }} - PYTEST_NUMPROCESSES: ${{ matrix.test-type == 'serial' && 0 || 'auto' }} steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout Source + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Timestamp - run: echo "TIMESTAMP=$(date -u "+%Y%m")" >> $GITHUB_ENV - shell: bash + - name: Hash + Timestamp + shell: bash # use bash to run date command + run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV - - name: Cache conda - uses: actions/cache@v3 + - name: Cache Conda + uses: actions/cache@v4 with: path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.TIMESTAMP }} + key: cache-${{ env.HASH }} - - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v2 + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3 with: - condarc-file: .\ci\github\.condarc - python-version: ${{ matrix.python-version }} + condarc-file: .github\condarc run-post: false # skip post cleanup - - name: Setup environment - shell: cmd /C CALL {0} - run: | - @echo on - CALL choco install visualstudio2017-workload-vctools || exit 1 - CALL conda install -q -y -c defaults ^ - --file .\tests\requirements.txt ^ - --file .\tests\requirements-windows.txt ^ - ${{ env.CONDA_CHANNEL_LABEL }}::conda || exit 1 - CALL pip install -e . --no-deps || exit 1 - - - name: Show info - run: | - conda info -a - conda list --show-channel-urls - - - name: Run tests - run: | - pytest ` - --color=yes ` - -v ` - -n "${{ env.PYTEST_NUMPROCESSES }}" ` - --basetemp "${{ runner.temp }}\${{ matrix.test-type}}" ` - --cov conda_build ` - --cov-append ` - --cov-branch ` - --cov-report xml ` - --replay-record-dir="${{ env.REPLAY_DIR }}" ` - --replay-base-name="${{ env.REPLAY_NAME }}" ` - -m "${{ env.PYTEST_MARKER }}" ` - .\tests - - - uses: codecov/codecov-action@v3 + - name: Choco Install + run: choco install visualstudio2017-workload-vctools + + - name: Conda Install + run: conda install + --yes + --file tests\requirements.txt + --file tests\requirements-${{ runner.os }}.txt + --file tests\requirements-ci.txt + python=${{ matrix.python-version }} + ${{ env.CONDA_CHANNEL_LABEL }}::conda + + # TODO: how can we remove this step? + - name: Install Self + run: pip install -e . + + - name: Conda Info + run: conda info --verbose + + - name: Conda List + run: conda list --show-channel-urls + + - name: Run Tests + # Windows is sensitive to long paths, using `--basetemp=${{ runner.temp }} to + # keep the test directories shorter + run: pytest + --cov=conda_build + --basetemp=${{ runner.temp }} + -n auto + -m "${{ env.PYTEST_MARKER }}" + + - name: Upload Coverage + uses: codecov/codecov-action@v4 with: - flags: ${{ matrix.test-type }},${{ matrix.python-version }},win-64 + flags: ${{ runner.os }},${{ runner.arch }},${{ matrix.python-version }},${{ matrix.test-type }} - - name: Upload Pytest Replay + - name: Upload Test Results if: '!cancelled()' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - path: ${{ env.REPLAY_DIR }} - name: ${{ env.REPLAY_NAME }}-${{ matrix.test-type }} + name: test-results-${{ env.HASH }} + path: | + .coverage + test-report.xml + retention-days: 1 # temporary, combined in aggregate below # macos test suite macos: @@ -262,6 +264,7 @@ jobs: runs-on: macos-11 defaults: run: + # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell shell: bash -el {0} strategy: fail-fast: false @@ -279,74 +282,71 @@ jobs: test-type: parallel env: CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }} - REPLAY_NAME: macOS-${{ matrix.conda-version }}-Py${{ matrix.python-version }} - REPLAY_DIR: ${{ github.workspace }}/pytest-replay PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }} - PYTEST_NUMPROCESSES: ${{ matrix.test-type == 'serial' && 0 || 'auto' }} steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Checkout Source + uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Timestamp - run: echo "TIMESTAMP=$(date -u "+%Y%m")" >> $GITHUB_ENV - shell: bash + - name: Hash + Timestamp + run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV - - name: Cache conda - uses: actions/cache@v3 + - name: Cache Conda + uses: actions/cache@v4 with: path: ~/conda_pkgs_dir - key: ${{ runner.os }}-conda-${{ env.TIMESTAMP }} + key: cache-${{ env.HASH }} - - name: Setup miniconda - uses: conda-incubator/setup-miniconda@v2 + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3 with: - condarc-file: ./ci/github/.condarc - python-version: ${{ matrix.python-version }} + condarc-file: .github/condarc run-post: false # skip post cleanup - - name: Setup environment - run: | - sudo xcode-select --switch /Applications/Xcode_11.7.app - conda install -q -y -c defaults \ - --file ./tests/requirements.txt \ - --file ./tests/requirements-macos.txt \ - ${{ env.CONDA_CHANNEL_LABEL }}::conda - pip install -e . --no-deps - - - name: Show info - run: | - conda info -a - conda list --show-channel-urls + - name: Xcode Install + run: sudo xcode-select --switch /Applications/Xcode_11.7.app - - name: Run tests - run: | - pytest \ - --color=yes \ - -v \ - -n "${{ env.PYTEST_NUMPROCESSES }}" \ - --basetemp "${{ runner.temp }}/${{ matrix.test-type }}" \ - --cov conda_build \ - --cov-append \ - --cov-branch \ - --cov-report xml \ - --replay-record-dir="${{ env.REPLAY_DIR }}" \ - --replay-base-name="${{ env.REPLAY_NAME }}" \ - -m "${{ env.PYTEST_MARKER }}" \ - ./tests - - - uses: codecov/codecov-action@v3 + - name: Conda Install + run: conda install + --yes + --file tests/requirements.txt + --file tests/requirements-${{ runner.os }}.txt + --file tests/requirements-ci.txt + python=${{ matrix.python-version }} + ${{ env.CONDA_CHANNEL_LABEL }}::conda + + # TODO: how can we remove this step? + - name: Install Self + run: pip install -e . + + - name: Conda Info + run: conda info --verbose + + - name: Conda List + run: conda list --show-channel-urls + + - name: Run Tests + run: pytest + --cov=conda_build + -n auto + -m "${{ env.PYTEST_MARKER }}" + + - name: Upload Coverage + uses: codecov/codecov-action@v4 with: - flags: ${{ matrix.test-type }},${{ matrix.python-version }},osx-64 + flags: ${{ runner.os }},${{ runner.arch }},${{ matrix.python-version }},${{ matrix.test-type }} - - name: Upload Pytest Replay + - name: Upload Test Results if: '!cancelled()' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: ${{ env.REPLAY_NAME }}-${{ matrix.test-type }} - path: ${{ env.REPLAY_DIR }} + name: test-results-${{ env.HASH }} + path: | + .coverage + test-report.xml + retention-days: 1 # temporary, combined in aggregate below # aggregate and upload aggregate: @@ -361,40 +361,37 @@ jobs: runs-on: ubuntu-latest steps: - - name: Download test results - uses: actions/download-artifact@v3 + - name: Download Artifacts + uses: actions/download-artifact@v4 - - name: Upload combined test results - # provides one downloadable archive of all .coverage/test-report.xml files - # of all matrix runs for further analysis. - uses: actions/upload-artifact@v3 + - name: Upload Combined Test Results + # provides one downloadable archive of all matrix run test results for further analysis + uses: actions/upload-artifact@v4 with: name: test-results-${{ github.sha }}-all - path: test-results-${{ github.sha }}-* - retention-days: 90 # default: 90 + path: test-results-* - name: Test Summary uses: test-summary/action@v2 with: - paths: ./test-results-${{ github.sha }}-**/test-report*.xml + paths: test-results-*/test-report.xml # required check analyze: - name: Analyze results needs: [linux, windows, macos, aggregate] if: '!cancelled()' runs-on: ubuntu-latest steps: - - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe + - name: Determine Success + uses: re-actors/alls-green@v1.2.2 with: + # permit jobs to be skipped if there are no code changes (see changes job) allowed-skips: ${{ toJSON(needs) }} jobs: ${{ toJSON(needs) }} # canary builds build: - name: Canary Build needs: [analyze] # only build canary build if # - prior steps succeeded, @@ -415,24 +412,28 @@ jobs: subdir: linux-64 - runner: macos-latest subdir: osx-64 + - runner: macos-14 + subdir: osx-arm64 - runner: windows-latest subdir: win-64 runs-on: ${{ matrix.runner }} steps: # Clean checkout of specific git ref needed for package metadata version # which needs env vars GIT_DESCRIBE_TAG and GIT_BUILD_STR: - - uses: actions/checkout@v3 + - name: Checkout Source + uses: actions/checkout@v4 with: ref: ${{ github.ref }} clean: true fetch-depth: 0 # Explicitly use Python 3.12 since each of the OSes has a different default Python - - uses: actions/setup-python@v4 + - name: Setup Python + uses: actions/setup-python@v4 with: python-version: '3.12' - - name: Detect label + - name: Detect Label shell: python run: | from pathlib import Path @@ -453,8 +454,8 @@ jobs: Path(environ["GITHUB_ENV"]).write_text(f"ANACONDA_ORG_LABEL={label}") - - name: Create and upload canary build - uses: conda/actions/canary-release@v23.7.0 + - name: Create & Upload + uses: conda/actions/canary-release@v24.2.0 with: package-name: ${{ github.event.repository.name }} subdir: ${{ matrix.subdir }} diff --git a/pyproject.toml b/pyproject.toml index e8cfc5e011..dd3e95dd56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,8 +133,16 @@ addopts = [ # "--store-durations", # not available yet "--strict-markers", "--tb=native", + "--xdoctest-modules", + "--xdoctest-style=google", "-vv", ] +doctest_optionflags = [ + "NORMALIZE_WHITESPACE", + "IGNORE_EXCEPTION_DETAIL", + "ALLOW_UNICODE", + "ELLIPSIS", +] markers = [ "serial: execute test serially (to avoid race conditions)", "slow: execute the slow tests if active", diff --git a/tests/requirements-linux.txt b/tests/requirements-Linux.txt similarity index 100% rename from tests/requirements-linux.txt rename to tests/requirements-Linux.txt diff --git a/tests/requirements-windows.txt b/tests/requirements-Windows.txt similarity index 100% rename from tests/requirements-windows.txt rename to tests/requirements-Windows.txt diff --git a/tests/requirements-ci.txt b/tests/requirements-ci.txt new file mode 100644 index 0000000000..23d78bb0b2 --- /dev/null +++ b/tests/requirements-ci.txt @@ -0,0 +1,19 @@ +anaconda-client +conda-forge::xdoctest +conda-verify +contextlib2 +coverage +cytoolz +git +numpy +perl +pip +pyflakes +pytest +pytest-cov +pytest-forked +pytest-mock +pytest-rerunfailures +pytest-xdist +ruamel.yaml +tomli # [py<3.11] for coverage pyproject.toml diff --git a/tests/requirements-macos.txt b/tests/requirements-macOS.txt similarity index 100% rename from tests/requirements-macos.txt rename to tests/requirements-macOS.txt diff --git a/tests/requirements.txt b/tests/requirements.txt index 5f96c8fd66..5e94d4111a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,37 +1,22 @@ beautifulsoup4 chardet conda >=23.5.0 -conda-forge::anaconda-client conda-index >=0.4.0 +conda-libmamba-solver # ensure we use libmamba conda-package-handling >=1.3 -conda-verify -contextlib2 -cytoolz filelock -git jinja2 jsonschema >=4.19 menuinst >=2 -numpy packaging -perl -pip pkginfo psutil py-lief -pyflakes -pytest -pytest-cov -pytest-forked -pytest-mock -pytest-replay -pytest-rerunfailures -pytest-xdist +python >=3.8 python-libarchive-c pytz +pyyaml requests -ripgrep -ruamel.yaml +ripgrep # for faster grep setuptools_scm # needed for devenv version detection -tomli # [py<3.11] for coverage pyproject.toml tqdm