Skip to content

Commit

Permalink
tests: Test relevant prereleases and allow to ignore releases (#4073)
Browse files Browse the repository at this point in the history 
If a package has a prerelease of a higher version than the highest
released stable version, make sure to test it, too. We consider alpha,
beta, and RC releases.

Also add an option to ignore specific releases (this is related to the
above since the script now pulls in two irrelevant alpha releases of
starlite).

Closes #4030

---------

Co-authored-by: Daniel Szoke <7881302+szokeasaurusrex@users.noreply.github.com>
  • Loading branch information
@sentrivana @szokeasaurusrex
sentrivana and szokeasaurusrex authored Feb 19, 2025
1 parent 67f0491 commit a3b6e5d
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 17 deletions.
35 changes: 35 additions & 0 deletions scripts/populate_tox/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,15 @@ integration_name: {
rule2: [package3, package4, ...],
},
"python": python_version_specifier,
"include": package_version_specifier,
}
```

When talking about version specifiers, we mean
[version specifiers as defined](https://packaging.python.org/en/latest/specifications/version-specifiers/#id5)
by the Python Packaging Authority. See also the actual implementation
in [packaging.specifiers](https://packaging.pypa.io/en/stable/specifiers.html).

### `package`

The name of the third party package as it's listed on PyPI. The script will
Expand Down Expand Up @@ -118,6 +124,35 @@ metadata or the SDK is explicitly not supporting some packages on specific
Python versions (because of, for example, broken context vars), the `python`
key can be used.

### `include`

Sometimes we only want to consider testing some specific versions of packages.
For example, the Starlite package has two alpha prereleases of version 2.0.0, but
we do not want to test these, since Starlite 2.0 was renamed to Litestar.

The value of the `include` key expects a version specifier defining which
versions should be considered for testing. For example, since we only want to test
versions below 2.x in Starlite, we can use

```python
"starlite": {
"include": "<2",
...
}
```

The `include` key can also be used to exclude a set of specific versions by using
`!=` version specifiers. For example, the Starlite restriction above could equivalently
be expressed like so:


```python
"starlite": {
"include": "!=2.0.0a1,!=2.0.0a2",
...
}
```


## How-Tos

Expand Down
1 change: 1 addition & 0 deletions scripts/populate_tox/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@
],
},
"python": "<=3.11",
"include": "!=2.0.0a1,!=2.0.0a2", # these are not relevant as there will never be a stable 2.0 release (starlite continues as litestar)
},
"statsig": {
"package": "statsig",
Expand Down
72 changes: 58 additions & 14 deletions scripts/populate_tox/populate_tox.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def fetch_release(package: str, version: Version) -> dict:

def _prefilter_releases(
integration: str, releases: dict[str, dict], older_than: Optional[datetime] = None
) -> list[Version]:
) -> tuple[list[Version], Optional[Version]]:
"""
Filter `releases`, removing releases that are for sure unsupported.
Expand All @@ -120,6 +120,10 @@ def _prefilter_releases(
they require additional API calls to be made. The purpose of this function is
to slim down the list so that we don't have to make more API calls than
necessary for releases that are for sure not supported.
The function returns a tuple with:
- the list of prefiltered releases
- an optional prerelease if there is one that should be tested
"""
min_supported = _MIN_VERSIONS.get(integration)
if min_supported is not None:
Expand All @@ -129,7 +133,14 @@ def _prefilter_releases(
f" {integration} doesn't have a minimum version defined in sentry_sdk/integrations/__init__.py. Consider defining one"
)

include_versions = None
if TEST_SUITE_CONFIG[integration].get("include") is not None:
include_versions = SpecifierSet(
TEST_SUITE_CONFIG[integration]["include"], prereleases=True
)

filtered_releases = []
last_prerelease = None

for release, data in releases.items():
if not data:
Expand All @@ -149,9 +160,15 @@ def _prefilter_releases(
if min_supported and version < min_supported:
continue

if version.is_prerelease or version.is_postrelease:
# TODO: consider the newest prerelease unless obsolete
# https://github.com/getsentry/sentry-python/issues/4030
if version.is_postrelease or version.is_devrelease:
continue

if include_versions is not None and version not in include_versions:
continue

if version.is_prerelease:
if last_prerelease is None or version > last_prerelease:
last_prerelease = version
continue

for i, saved_version in enumerate(filtered_releases):
Expand All @@ -166,18 +183,30 @@ def _prefilter_releases(
else:
filtered_releases.append(version)

return sorted(filtered_releases)
filtered_releases.sort()

# Check if the latest prerelease is relevant (i.e., it's for a version higher
# than the last released version); if not, don't consider it
if last_prerelease is not None:
if not filtered_releases or last_prerelease > filtered_releases[-1]:
return filtered_releases, last_prerelease

return filtered_releases, None


def get_supported_releases(
integration: str, pypi_data: dict, older_than: Optional[datetime] = None
) -> list[Version]:
) -> tuple[list[Version], Optional[Version]]:
"""
Get a list of releases that are currently supported by the SDK.
This takes into account a handful of parameters (Python support, the lowest
version we've defined for the framework, the date of the release).
We return the list of supported releases and optionally also the newest
prerelease, if it should be tested (meaning it's for a version higher than
the current stable version).
If an `older_than` timestamp is provided, no release newer than that will be
considered.
"""
Expand All @@ -186,7 +215,9 @@ def get_supported_releases(
# Get a consolidated list without taking into account Python support yet
# (because that might require an additional API call for some
# of the releases)
releases = _prefilter_releases(integration, pypi_data["releases"], older_than)
releases, latest_prerelease = _prefilter_releases(
integration, pypi_data["releases"], older_than
)

# Determine Python support
expected_python_versions = TEST_SUITE_CONFIG[integration].get("python")
Expand All @@ -210,14 +241,18 @@ def _supports_lowest(release: Version) -> bool:
# version(s) that we do, cut off the rest
releases = releases[i:]

return releases
return releases, latest_prerelease


def pick_releases_to_test(releases: list[Version]) -> list[Version]:
def pick_releases_to_test(
releases: list[Version], last_prerelease: Optional[Version]
) -> list[Version]:
"""Pick a handful of releases to test from a sorted list of supported releases."""
# If the package has majors (or major-like releases, even if they don't do
# semver), we want to make sure we're testing them all. If not, we just pick
# the oldest, the newest, and a couple in between.
#
# If there is a relevant prerelease, also test that in addition to the above.
has_majors = len(set([v.major for v in releases])) > 1
filtered_releases = set()

Expand Down Expand Up @@ -252,7 +287,11 @@ def pick_releases_to_test(releases: list[Version]) -> list[Version]:
releases[-1], # latest
}

return sorted(filtered_releases)
filtered_releases = sorted(filtered_releases)
if last_prerelease is not None:
filtered_releases.append(last_prerelease)

return filtered_releases


def supported_python_versions(
Expand Down Expand Up @@ -553,19 +592,24 @@ def main(fail_on_changes: bool = False) -> None:
pypi_data = fetch_package(package)

# Get the list of all supported releases
# If in check mode, ignore releases newer than `last_updated`

# If in fail-on-changes mode, ignore releases newer than `last_updated`
older_than = last_updated if fail_on_changes else None
releases = get_supported_releases(integration, pypi_data, older_than)

releases, latest_prerelease = get_supported_releases(
integration, pypi_data, older_than
)

if not releases:
print(" Found no supported releases.")
continue

_compare_min_version_with_defined(integration, releases)

# Pick a handful of the supported releases to actually test against
# and fetch the PYPI data for each to determine which Python versions
# and fetch the PyPI data for each to determine which Python versions
# to test it on
test_releases = pick_releases_to_test(releases)
test_releases = pick_releases_to_test(releases, latest_prerelease)

for release in test_releases:
_add_python_versions_to_release(integration, package, release)
Expand Down
10 changes: 7 additions & 3 deletions tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
# The file (and all resulting CI YAMLs) then need to be regenerated via
# "scripts/generate-test-files.sh".
#
# Last generated: 2025-02-18T12:57:32.874168+00:00
# Last generated: 2025-02-19T12:41:15.689786+00:00

[tox]
requires =
Expand Down Expand Up @@ -211,10 +211,11 @@ envlist =
{py3.8,py3.10,py3.11}-ariadne-v0.20.1
{py3.8,py3.11,py3.12}-ariadne-v0.22
{py3.8,py3.11,py3.12}-ariadne-v0.24.0
{py3.8,py3.11,py3.12}-ariadne-v0.25.2
{py3.9,py3.12,py3.13}-ariadne-v0.26.0

{py3.6,py3.9,py3.10}-gql-v3.4.1
{py3.7,py3.11,py3.12}-gql-v3.5.0
{py3.9,py3.12,py3.13}-gql-v3.6.0b4

{py3.6,py3.9,py3.10}-graphene-v3.3
{py3.8,py3.12,py3.13}-graphene-v3.4.3
Expand All @@ -236,6 +237,7 @@ envlist =
{py3.6,py3.7,py3.8}-celery-v4.4.7
{py3.6,py3.7,py3.8}-celery-v5.0.5
{py3.8,py3.11,py3.12}-celery-v5.4.0
{py3.8,py3.12,py3.13}-celery-v5.5.0rc4

{py3.6,py3.7}-dramatiq-v1.9.0
{py3.6,py3.8,py3.9}-dramatiq-v1.12.3
Expand Down Expand Up @@ -592,13 +594,14 @@ deps =
ariadne-v0.20.1: ariadne==0.20.1
ariadne-v0.22: ariadne==0.22
ariadne-v0.24.0: ariadne==0.24.0
ariadne-v0.25.2: ariadne==0.25.2
ariadne-v0.26.0: ariadne==0.26.0
ariadne: fastapi
ariadne: flask
ariadne: httpx

gql-v3.4.1: gql[all]==3.4.1
gql-v3.5.0: gql[all]==3.5.0
gql-v3.6.0b4: gql[all]==3.6.0b4

graphene-v3.3: graphene==3.3
graphene-v3.4.3: graphene==3.4.3
Expand Down Expand Up @@ -630,6 +633,7 @@ deps =
celery-v4.4.7: celery==4.4.7
celery-v5.0.5: celery==5.0.5
celery-v5.4.0: celery==5.4.0
celery-v5.5.0rc4: celery==5.5.0rc4
celery: newrelic
celery: redis
py3.7-celery: importlib-metadata<5.0
Expand Down

0 comments on commit a3b6e5d

Please sign in to comment.