From 4183c58a88e41062627473c900da2aefe6c3e200 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 13 Mar 2024 09:27:02 -0500 Subject: [PATCH 1/8] Add CodSpeed --- .github/workflows/tests.yml | 73 ++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 090c389a6b..feca0b1794 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -161,6 +161,75 @@ jobs: test-report.xml retention-days: 1 # temporary, combined in aggregate below + # linux benchmarks + linux-benchmarks: + # only run test suite if there are code changes + needs: changes + if: needs.changes.outputs.code == 'true' + + runs-on: ubuntu-latest + defaults: + run: + # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell + shell: bash -el {0} # bash exit immediately on error + login shell + strategy: + fail-fast: false + matrix: + python-version: ['3.12'] + + steps: + - name: Checkout Source + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Hash + Timestamp + run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-benchmark-$(date -u "+%Y%m")" >> $GITHUB_ENV + + - name: Cache Conda + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: cache-${{ env.HASH }} + + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + condarc-file: .github/condarc + run-post: false # skip post cleanup + + - name: Conda Install + run: conda install + --yes + --file tests/requirements.txt + --file tests/requirements-${{ runner.os }}.txt + --file tests/requirements-ci.txt + python=${{ matrix.python-version }} + ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }} + + - name: Install CodSpeed + run: pip install git+https://github.com/kenodegard/pytest-codspeed.git@fix-outerr-redirects#egg=pytest-codspeed + + # TODO: how can we remove this step? + - name: Install Self + run: pip install -e . + + - name: Conda Info + # view test env info (not base) + run: python -m conda info --verbose + + - name: Conda Config + run: conda config --show-sources + + - name: Conda List + run: conda list --show-channel-urls + + - name: Run Benchmarks + uses: CodSpeedHQ/action@v2 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: $CONDA/envs/test/bin/pytest --codspeed + # windows test suite windows: # only run test suite if there are code changes @@ -351,7 +420,7 @@ jobs: # aggregate and upload aggregate: # only aggregate test suite if there are code changes - needs: [changes, linux, windows, macos] + needs: [changes, linux, linux-benchmarks, windows, macos] if: >- !cancelled() && ( @@ -378,7 +447,7 @@ jobs: # required check analyze: - needs: [linux, windows, macos, aggregate] + needs: [linux, linux-benchmarks, windows, macos, aggregate] if: '!cancelled()' runs-on: ubuntu-latest From 40019f65b2b4e790eadc3feb81b48ff22981997a Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 13 Mar 2024 09:15:45 -0500 Subject: [PATCH 2/8] Indent test_select_lines & mark as benchmark --- tests/test_metadata.py | 96 ++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 37 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index b176d4103d..03a94cb0da 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -53,50 +53,72 @@ def test_uses_vcs_in_metadata(testing_workdir, testing_metadata): assert not testing_metadata.uses_vcs_in_build +@pytest.mark.benchmark def test_select_lines(): - lines = """ -test -test [abc] no -test [abc] # no - -test [abc] - 'quoted # [abc] ' - "quoted # [abc] yes " -test # stuff [abc] yes -test {{ JINJA_VAR[:2] }} -test {{ JINJA_VAR[:2] }} # stuff [abc] yes -test {{ JINJA_VAR[:2] }} # stuff yes [abc] -test {{ JINJA_VAR[:2] }} # [abc] stuff yes -{{ environ["test"] }} # [abc] -""" + lines = ( + "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test ' ", + ' " test " ', + "", + "# comment line", + "test [abc]", + " 'quoted # [abc] '", + ' "quoted # [abc] yes "', + "test # stuff [abc] yes", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }} # stuff [abc] yes", + "test {{ JINJA_VAR[:2] }} # stuff yes [abc]", + "test {{ JINJA_VAR[:2] }} # [abc] stuff yes", + '{{ environ["test"] }} # [abc]', + ) + ) + + "\n" + ) assert ( select_lines(lines, {"abc": True}, variants_in_place=True) - == """ -test -test [abc] no -test [abc] # no - -test - 'quoted' - "quoted" -test -test {{ JINJA_VAR[:2] }} -test {{ JINJA_VAR[:2] }} -test {{ JINJA_VAR[:2] }} -test {{ JINJA_VAR[:2] }} -{{ environ["test"] }} -""" + == "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test '", + ' " test "', + "", + "test", + " 'quoted'", + ' "quoted"', + "test", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + '{{ environ["test"] }}', + ) + ) + + "\n" ) assert ( select_lines(lines, {"abc": False}, variants_in_place=True) - == """ -test -test [abc] no -test [abc] # no - -test {{ JINJA_VAR[:2] }} -""" + == "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test '", + ' " test "', + "", + "test {{ JINJA_VAR[:2] }}", + ) + ) + + "\n" ) From ac09844bf0a75f9103f2f9d869ba8321648981e4 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 14 Mar 2024 14:08:31 -0500 Subject: [PATCH 3/8] Fix test_which_package_battery --- tests/test_inspect_pkg.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_inspect_pkg.py b/tests/test_inspect_pkg.py index 2f35bd3b0e..1ac203723e 100644 --- a/tests/test_inspect_pkg.py +++ b/tests/test_inspect_pkg.py @@ -207,6 +207,14 @@ def test_which_package(tmp_path: Path): @pytest.mark.benchmark def test_which_package_battery(tmp_path: Path): # regression: https://github.com/conda/conda-build/issues/5126 + + # NOTE: CodSpeed on Python 3.12+ activates the stack profiler trampoline backend + # and thus runs the test twice (once without profiling and once with profiling), + # unfortunately this means that on the second iteration tmp_path is no longer empty + # so we create a randomized unique directory to compensate + tmp_path = tmp_path / uuid4().hex + tmp_path.mkdir() + # create a dummy environment (tmp_path / "conda-meta").mkdir() (tmp_path / "conda-meta" / "history").touch() From 234a59345988367fd69189763e61340b32e92616 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 14 Mar 2024 15:17:34 -0500 Subject: [PATCH 4/8] Add test_select_lines_battery --- tests/test_metadata.py | 53 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 03a94cb0da..a216457c81 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -53,7 +53,6 @@ def test_uses_vcs_in_metadata(testing_workdir, testing_metadata): assert not testing_metadata.uses_vcs_in_build -@pytest.mark.benchmark def test_select_lines(): lines = ( "\n".join( @@ -122,6 +121,58 @@ def test_select_lines(): ) +@pytest.mark.benchmark +def test_select_lines_battery(): + test_foo = "test [foo]" + test_bar = "test [bar]" + test_baz = "test [baz]" + test_foo_and_bar = "test [foo and bar]" + test_foo_and_baz = "test [foo and baz]" + test_foo_or_bar = "test [foo or bar]" + test_foo_or_baz = "test [foo or baz]" + + lines = "\n".join( + ( + test_foo, + test_bar, + test_baz, + test_foo_and_bar, + test_foo_and_baz, + test_foo_or_bar, + test_foo_or_baz, + ) + * 100 + ) + + for _ in range(100): + for foo in (True, False): + for bar in (True, False): + for baz in (True, False): + assert ( + select_lines( + lines, + {"foo": foo, "bar": bar, "baz": baz}, + variants_in_place=True, + ) + == "\n".join( + filter( + None, + ( + foo and "test", + bar and "test", + baz and "test", + (foo and bar) and "test", + (foo and baz) and "test", + (foo or bar) and "test", + (foo or baz) and "test", + ) + * 100, + ) + ) + + "\n" + ) + + def test_disallow_leading_period_in_version(testing_metadata): testing_metadata.meta["package"]["version"] = ".ste.ve" testing_metadata.final = True From 489383faae6f2cd01cefae9c567e95252e2915d8 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 14 Mar 2024 15:28:35 -0500 Subject: [PATCH 5/8] Add news --- news/5233-enable-codspeed | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 news/5233-enable-codspeed diff --git a/news/5233-enable-codspeed b/news/5233-enable-codspeed new file mode 100644 index 0000000000..efb32df4d1 --- /dev/null +++ b/news/5233-enable-codspeed @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* Enable CodSpeed benchmarks for select tests. (#5233) From e7f1b724244eeca046c183007c72de8dc2335e7c Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 14 Mar 2024 16:02:35 -0500 Subject: [PATCH 6/8] Trailing --- tests/test_metadata.py | 152 ++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 84 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index a216457c81..f4fa57a039 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -5,6 +5,7 @@ import os import subprocess import sys +from itertools import product from typing import TYPE_CHECKING import pytest @@ -54,70 +55,62 @@ def test_uses_vcs_in_metadata(testing_workdir, testing_metadata): def test_select_lines(): - lines = ( - "\n".join( - ( - "", - "test", - "test [abc] no", - "test [abc] # no", - " ' test ' ", - ' " test " ', - "", - "# comment line", - "test [abc]", - " 'quoted # [abc] '", - ' "quoted # [abc] yes "', - "test # stuff [abc] yes", - "test {{ JINJA_VAR[:2] }}", - "test {{ JINJA_VAR[:2] }} # stuff [abc] yes", - "test {{ JINJA_VAR[:2] }} # stuff yes [abc]", - "test {{ JINJA_VAR[:2] }} # [abc] stuff yes", - '{{ environ["test"] }} # [abc]', - ) + lines = "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test ' ", + ' " test " ', + "", + "# comment line", + "test [abc]", + " 'quoted # [abc] '", + ' "quoted # [abc] yes "', + "test # stuff [abc] yes", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }} # stuff [abc] yes", + "test {{ JINJA_VAR[:2] }} # stuff yes [abc]", + "test {{ JINJA_VAR[:2] }} # [abc] stuff yes", + '{{ environ["test"] }} # [abc]', + "", # trailing newline ) - + "\n" ) - assert ( - select_lines(lines, {"abc": True}, variants_in_place=True) - == "\n".join( - ( - "", - "test", - "test [abc] no", - "test [abc] # no", - " ' test '", - ' " test "', - "", - "test", - " 'quoted'", - ' "quoted"', - "test", - "test {{ JINJA_VAR[:2] }}", - "test {{ JINJA_VAR[:2] }}", - "test {{ JINJA_VAR[:2] }}", - "test {{ JINJA_VAR[:2] }}", - '{{ environ["test"] }}', - ) + assert select_lines(lines, {"abc": True}, variants_in_place=True) == "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test '", + ' " test "', + "", + "test", + " 'quoted'", + ' "quoted"', + "test", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + '{{ environ["test"] }}', + "", # trailing newline ) - + "\n" ) - assert ( - select_lines(lines, {"abc": False}, variants_in_place=True) - == "\n".join( - ( - "", - "test", - "test [abc] no", - "test [abc] # no", - " ' test '", - ' " test "', - "", - "test {{ JINJA_VAR[:2] }}", - ) + assert select_lines(lines, {"abc": False}, variants_in_place=True) == "\n".join( + ( + "", + "test", + "test [abc] no", + "test [abc] # no", + " ' test '", + ' " test "', + "", + "test {{ JINJA_VAR[:2] }}", + "", # trailing newline ) - + "\n" ) @@ -145,32 +138,23 @@ def test_select_lines_battery(): ) for _ in range(100): - for foo in (True, False): - for bar in (True, False): - for baz in (True, False): - assert ( - select_lines( - lines, - {"foo": foo, "bar": bar, "baz": baz}, - variants_in_place=True, - ) - == "\n".join( - filter( - None, - ( - foo and "test", - bar and "test", - baz and "test", - (foo and bar) and "test", - (foo and baz) and "test", - (foo or bar) and "test", - (foo or baz) and "test", - ) - * 100, - ) - ) - + "\n" - ) + for foo, bar, baz in product((True, False), repeat=3): + namespace = {"foo": foo, "bar": bar, "baz": baz} + selection = ( + ["test"] + * ( + foo + + bar + + baz + + (foo and bar) + + (foo and baz) + + (foo or bar) + + (foo or baz) + ) + * 100 + ) + selection = "\n".join(selection) + "\n" # trailing newline + assert select_lines(lines, namespace, variants_in_place=True) == selection def test_disallow_leading_period_in_version(testing_metadata): From ec19c8ec3ca06656f4c0f74dfc60e433520504f1 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 14 Mar 2024 16:30:37 -0500 Subject: [PATCH 7/8] Limit test_which_package_battery --- tests/test_inspect_pkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_inspect_pkg.py b/tests/test_inspect_pkg.py index 1ac203723e..dae6d7f6ca 100644 --- a/tests/test_inspect_pkg.py +++ b/tests/test_inspect_pkg.py @@ -222,7 +222,7 @@ def test_which_package_battery(tmp_path: Path): # dummy packages with files removed = [] - for _ in range(100): + for _ in range(10): name = f"package_{uuid4().hex}" # mock a package with 100 files From 108a5f20ff244fef60e8a0f6b0076f2d5e6de205 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 14 Mar 2024 17:03:55 -0500 Subject: [PATCH 8/8] Limit test_select_lines_battery --- tests/test_metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index f4fa57a039..0f6da9b089 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -134,10 +134,10 @@ def test_select_lines_battery(): test_foo_or_bar, test_foo_or_baz, ) - * 100 + * 10 ) - for _ in range(100): + for _ in range(10): for foo, bar, baz in product((True, False), repeat=3): namespace = {"foo": foo, "bar": bar, "baz": baz} selection = ( @@ -151,7 +151,7 @@ def test_select_lines_battery(): + (foo or bar) + (foo or baz) ) - * 100 + * 10 ) selection = "\n".join(selection) + "\n" # trailing newline assert select_lines(lines, namespace, variants_in_place=True) == selection