Enable CodSpeed and add select lines benchmark (#5233)

* Add CodSpeed
* Indent test_select_lines
* Add test_select_lines_battery
* Fix & limit test_which_package_battery

.github/workflows/tests.yml

@@ -162,6 +162,75 @@ jobs:
             test-report.xml
           retention-days: 1  # temporary, combined in aggregate below
 
+  # linux benchmarks
+  linux-benchmarks:
+    # only run test suite if there are code changes
+    needs: changes
+    if: needs.changes.outputs.code == 'true'
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        # https://github.com/conda-incubator/setup-miniconda#use-a-default-shell
+        shell: bash -el {0}  # bash exit immediately on error + login shell
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.12']
+
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Hash + Timestamp
+        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-benchmark-$(date -u "+%Y%m")" >> $GITHUB_ENV
+
+      - name: Cache Conda
+        uses: actions/cache@v4
+        with:
+          path: ~/conda_pkgs_dir
+          key: cache-${{ env.HASH }}
+
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          condarc-file: .github/condarc
+          run-post: false  # skip post cleanup
+
+      - name: Conda Install
+        run: conda install
+          --yes
+          --file tests/requirements.txt
+          --file tests/requirements-${{ runner.os }}.txt
+          --file tests/requirements-ci.txt
+          python=${{ matrix.python-version }}
+          ${{ env.CONDA_CHANNEL_LABEL }}${{ env.CONDA_VERSION }}
+
+      - name: Install CodSpeed
+        run: pip install git+https://github.com/kenodegard/pytest-codspeed.git@fix-outerr-redirects#egg=pytest-codspeed
+
+      # TODO: how can we remove this step?
+      - name: Install Self
+        run: pip install -e .
+
+      - name: Conda Info
+        # view test env info (not base)
+        run: python -m conda info --verbose
+
+      - name: Conda Config
+        run: conda config --show-sources
+
+      - name: Conda List
+        run: conda list --show-channel-urls
+
+      - name: Run Benchmarks
+        uses: CodSpeedHQ/action@v2
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: $CONDA/envs/test/bin/pytest --codspeed
+
   # windows test suite
   windows:
     # only run test suite if there are code changes
@@ -354,7 +423,7 @@ jobs:
   # aggregate and upload
   aggregate:
     # only aggregate test suite if there are code changes
-    needs: [changes, linux, windows, macos]
+    needs: [changes, linux, linux-benchmarks, windows, macos]
     if: >-
       !cancelled()
       && (
@@ -381,7 +450,7 @@ jobs:
 
   # required check
   analyze:
-    needs: [linux, windows, macos, aggregate]
+    needs: [linux, linux-benchmarks, windows, macos, aggregate]
     if: '!cancelled()'
 
     runs-on: ubuntu-latest

news/5233-enable-codspeed

@@ -0,0 +1,19 @@
+### Enhancements
+
+* <news item>
+
+### Bug fixes
+
+* <news item>
+
+### Deprecations
+
+* <news item>
+
+### Docs
+
+* <news item>
+
+### Other
+
+* Enable CodSpeed benchmarks for select tests. (#5233)

tests/test_inspect_pkg.py

@@ -207,14 +207,22 @@ def test_which_package(tmp_path: Path):
 @pytest.mark.benchmark
 def test_which_package_battery(tmp_path: Path):
     # regression: https://github.com/conda/conda-build/issues/5126
+
+    # NOTE: CodSpeed on Python 3.12+ activates the stack profiler trampoline backend
+    # and thus runs the test twice (once without profiling and once with profiling),
+    # unfortunately this means that on the second iteration tmp_path is no longer empty
+    # so we create a randomized unique directory to compensate
+    tmp_path = tmp_path / uuid4().hex
+    tmp_path.mkdir()
+
     # create a dummy environment
     (tmp_path / "conda-meta").mkdir()
     (tmp_path / "conda-meta" / "history").touch()
     (tmp_path / "lib").mkdir()
 
     # dummy packages with files
     removed = []
-    for _ in range(100):
+    for _ in range(10):
         name = f"package_{uuid4().hex}"
 
         # mock a package with 100 files

tests/test_metadata.py

@@ -5,6 +5,7 @@
 import os
 import subprocess
 import sys
+from itertools import product
 from typing import TYPE_CHECKING
 
 import pytest
@@ -54,52 +55,108 @@ def test_uses_vcs_in_metadata(testing_workdir, testing_metadata):
 
 
 def test_select_lines():
-    lines = """
-test
-test [abc] no
-test [abc] # no
-
-test [abc]
- 'quoted # [abc] '
- "quoted # [abc] yes "
-test # stuff [abc] yes
-test {{ JINJA_VAR[:2] }}
-test {{ JINJA_VAR[:2] }} # stuff [abc] yes
-test {{ JINJA_VAR[:2] }} # stuff yes [abc]
-test {{ JINJA_VAR[:2] }} # [abc] stuff yes
-{{ environ["test"] }}  # [abc]
-"""
+    lines = "\n".join(
+        (
+            "",
+            "test",
+            "test [abc] no",
+            "test [abc] # no",
+            " ' test ' ",
+            ' " test " ',
+            "",
+            "# comment line",
+            "test [abc]",
+            " 'quoted # [abc] '",
+            ' "quoted # [abc] yes "',
+            "test # stuff [abc] yes",
+            "test {{ JINJA_VAR[:2] }}",
+            "test {{ JINJA_VAR[:2] }} # stuff [abc] yes",
+            "test {{ JINJA_VAR[:2] }} # stuff yes [abc]",
+            "test {{ JINJA_VAR[:2] }} # [abc] stuff yes",
+            '{{ environ["test"] }}  # [abc]',
+            "",  # trailing newline
+        )
+    )
 
-    assert (
-        select_lines(lines, {"abc": True}, variants_in_place=True)
-        == """
-test
-test [abc] no
-test [abc] # no
-
-test
- 'quoted'
- "quoted"
-test
-test {{ JINJA_VAR[:2] }}
-test {{ JINJA_VAR[:2] }}
-test {{ JINJA_VAR[:2] }}
-test {{ JINJA_VAR[:2] }}
-{{ environ["test"] }}
-"""
+    assert select_lines(lines, {"abc": True}, variants_in_place=True) == "\n".join(
+        (
+            "",
+            "test",
+            "test [abc] no",
+            "test [abc] # no",
+            " ' test '",
+            ' " test "',
+            "",
+            "test",
+            " 'quoted'",
+            ' "quoted"',
+            "test",
+            "test {{ JINJA_VAR[:2] }}",
+            "test {{ JINJA_VAR[:2] }}",
+            "test {{ JINJA_VAR[:2] }}",
+            "test {{ JINJA_VAR[:2] }}",
+            '{{ environ["test"] }}',
+            "",  # trailing newline
+        )
     )
-    assert (
-        select_lines(lines, {"abc": False}, variants_in_place=True)
-        == """
-test
-test [abc] no
-test [abc] # no
-
-test {{ JINJA_VAR[:2] }}
-"""
+    assert select_lines(lines, {"abc": False}, variants_in_place=True) == "\n".join(
+        (
+            "",
+            "test",
+            "test [abc] no",
+            "test [abc] # no",
+            " ' test '",
+            ' " test "',
+            "",
+            "test {{ JINJA_VAR[:2] }}",
+            "",  # trailing newline
+        )
     )
 
 
+@pytest.mark.benchmark
+def test_select_lines_battery():
+    test_foo = "test [foo]"
+    test_bar = "test [bar]"
+    test_baz = "test [baz]"
+    test_foo_and_bar = "test [foo and bar]"
+    test_foo_and_baz = "test [foo and baz]"
+    test_foo_or_bar = "test [foo or bar]"
+    test_foo_or_baz = "test [foo or baz]"
+
+    lines = "\n".join(
+        (
+            test_foo,
+            test_bar,
+            test_baz,
+            test_foo_and_bar,
+            test_foo_and_baz,
+            test_foo_or_bar,
+            test_foo_or_baz,
+        )
+        * 10
+    )
+
+    for _ in range(10):
+        for foo, bar, baz in product((True, False), repeat=3):
+            namespace = {"foo": foo, "bar": bar, "baz": baz}
+            selection = (
+                ["test"]
+                * (
+                    foo
+                    + bar
+                    + baz
+                    + (foo and bar)
+                    + (foo and baz)
+                    + (foo or bar)
+                    + (foo or baz)
+                )
+                * 10
+            )
+            selection = "\n".join(selection) + "\n"  # trailing newline
+            assert select_lines(lines, namespace, variants_in_place=True) == selection
+
+
 def test_disallow_leading_period_in_version(testing_metadata):
     testing_metadata.meta["package"]["version"] = ".ste.ve"
     testing_metadata.final = True