diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 090c389a6b..feca0b1794 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -161,6 +161,75 @@ jobs: test-report.xml retention-days: 1 # temporary, combined in aggregate below + # linux benchmarks + linux-benchmarks: + # only run test suite if there are code changes + needs: changes + if: needs.changes.outputs.code == 'true' + + runs-on: ubuntu-latest + defaults: + run: + # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell + shell: bash -el {0} # bash exit immediately on error + login shell + strategy: + fail-fast: false + matrix: + python-version: ['3.12'] + + steps: + - name: Checkout Source + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Hash + Timestamp + run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-benchmark-$(date -u "+%Y%m")" >> $GITHUB_ENV + + - name: Cache Conda + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: cache-${{ env.HASH }} + + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + condarc-file: .github/condarc + run-post: false # skip post cleanup + + - name: Conda Install + run: conda install + --yes + --file tests/requirements.txt + --file tests/requirements-${{ runner.os }}.txt + --file tests/requirements-ci.txt + python=${{ matrix.python-version }} + ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }} + + - name: Install CodSpeed + run: pip install git+https://github.com/kenodegard/pytest-codspeed.git@fix-outerr-redirects#egg=pytest-codspeed + + # TODO: how can we remove this step? + - name: Install Self + run: pip install -e . + + - name: Conda Info + # view test env info (not base) + run: python -m conda info --verbose + + - name: Conda Config + run: conda config --show-sources + + - name: Conda List + run: conda list --show-channel-urls + + - name: Run Benchmarks + uses: CodSpeedHQ/action@v2 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: $CONDA/envs/test/bin/pytest --codspeed + # windows test suite windows: # only run test suite if there are code changes @@ -351,7 +420,7 @@ jobs: # aggregate and upload aggregate: # only aggregate test suite if there are code changes - needs: [changes, linux, windows, macos] + needs: [changes, linux, linux-benchmarks, windows, macos] if: >- !cancelled() && ( @@ -378,7 +447,7 @@ jobs: # required check analyze: - needs: [linux, windows, macos, aggregate] + needs: [linux, linux-benchmarks, windows, macos, aggregate] if: '!cancelled()' runs-on: ubuntu-latest diff --git a/news/5233-enable-codspeed b/news/5233-enable-codspeed new file mode 100644 index 0000000000..efb32df4d1 --- /dev/null +++ b/news/5233-enable-codspeed @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* Enable CodSpeed benchmarks for select tests. (#5233) diff --git a/tests/test_inspect_pkg.py b/tests/test_inspect_pkg.py index 2f35bd3b0e..dae6d7f6ca 100644 --- a/tests/test_inspect_pkg.py +++ b/tests/test_inspect_pkg.py @@ -207,6 +207,14 @@ def test_which_package(tmp_path: Path): @pytest.mark.benchmark def test_which_package_battery(tmp_path: Path): # regression: https://github.com/conda/conda-build/issues/5126 + + # NOTE: CodSpeed on Python 3.12+ activates the stack profiler trampoline backend + # and thus runs the test twice (once without profiling and once with profiling), + # unfortunately this means that on the second iteration tmp_path is no longer empty + # so we create a randomized unique directory to compensate + tmp_path = tmp_path / uuid4().hex + tmp_path.mkdir() + # create a dummy environment (tmp_path / "conda-meta").mkdir() (tmp_path / "conda-meta" / "history").touch() @@ -214,7 +222,7 @@ def test_which_package_battery(tmp_path: Path): # dummy packages with files removed = [] - for _ in range(100): + for _ in range(10): name = f"package_{uuid4().hex}" # mock a package with 100 files diff --git a/tests/test_metadata.py b/tests/test_metadata.py index b176d4103d..0f6da9b089 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -5,6 +5,7 @@ import os import subprocess import sys +from itertools import product from typing import TYPE_CHECKING import pytest @@ -54,52 +55,108 @@ def test_uses_vcs_in_metadata(testing_workdir, testing_metadata): def test_select_lines(): - lines = """ -test -test [abc] no -test [abc] # no - -test [abc] - 'quoted # [abc] ' - "quoted # [abc] yes " -test # stuff [abc] yes -test {{ JINJA_VAR[:2] }} -test {{ JINJA_VAR[:2] }} # stuff [abc] yes -test {{ JINJA_VAR[:2] }} # stuff yes [abc] -test {{ JINJA_VAR[:2] }} # [abc] stuff yes -{{ environ["test"] }} # [abc] -""" + lines = "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test ' ", + ' " test " ', + "", + "# comment line", + "test [abc]", + " 'quoted # [abc] '", + ' "quoted # [abc] yes "', + "test # stuff [abc] yes", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }} # stuff [abc] yes", + "test {{ JINJA_VAR[:2] }} # stuff yes [abc]", + "test {{ JINJA_VAR[:2] }} # [abc] stuff yes", + '{{ environ["test"] }} # [abc]', + "", # trailing newline + ) + ) - assert ( - select_lines(lines, {"abc": True}, variants_in_place=True) - == """ -test -test [abc] no -test [abc] # no - -test - 'quoted' - "quoted" -test -test {{ JINJA_VAR[:2] }} -test {{ JINJA_VAR[:2] }} -test {{ JINJA_VAR[:2] }} -test {{ JINJA_VAR[:2] }} -{{ environ["test"] }} -""" + assert select_lines(lines, {"abc": True}, variants_in_place=True) == "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test '", + ' " test "', + "", + "test", + " 'quoted'", + ' "quoted"', + "test", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + '{{ environ["test"] }}', + "", # trailing newline + ) ) - assert ( - select_lines(lines, {"abc": False}, variants_in_place=True) - == """ -test -test [abc] no -test [abc] # no - -test {{ JINJA_VAR[:2] }} -""" + assert select_lines(lines, {"abc": False}, variants_in_place=True) == "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test '", + ' " test "', + "", + "test {{ JINJA_VAR[:2] }}", + "", # trailing newline + ) ) +@pytest.mark.benchmark +def test_select_lines_battery(): + test_foo = "test [foo]" + test_bar = "test [bar]" + test_baz = "test [baz]" + test_foo_and_bar = "test [foo and bar]" + test_foo_and_baz = "test [foo and baz]" + test_foo_or_bar = "test [foo or bar]" + test_foo_or_baz = "test [foo or baz]" + + lines = "\n".join( + ( + test_foo, + test_bar, + test_baz, + test_foo_and_bar, + test_foo_and_baz, + test_foo_or_bar, + test_foo_or_baz, + ) + * 10 + ) + + for _ in range(10): + for foo, bar, baz in product((True, False), repeat=3): + namespace = {"foo": foo, "bar": bar, "baz": baz} + selection = ( + ["test"] + * ( + foo + + bar + + baz + + (foo and bar) + + (foo and baz) + + (foo or bar) + + (foo or baz) + ) + * 10 + ) + selection = "\n".join(selection) + "\n" # trailing newline + assert select_lines(lines, namespace, variants_in_place=True) == selection + + def test_disallow_leading_period_in_version(testing_metadata): testing_metadata.meta["package"]["version"] = ".ste.ve" testing_metadata.final = True