From 4efef5270ab257f260f71fd9d4744800fe30d706 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Jun 2024 09:52:27 -0400 Subject: [PATCH 1/7] Add JSON encoder This command produces JSON output to be consumed by shell scripts via the jq command. --- pyproject.toml | 3 + src/rapids_metadata/json.py | 90 +++++++++++++ tests/metadata/test_json.py | 248 ++++++++++++++++++++++++++++++++++++ 3 files changed, 341 insertions(+) create mode 100644 src/rapids_metadata/json.py create mode 100644 tests/metadata/test_json.py diff --git a/pyproject.toml b/pyproject.toml index 4bbd077..f03679a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,9 @@ classifiers = [ ] requires-python = ">=3.9" +[project.scripts] +rapids-metadata-json = "rapids_metadata.json:main" + [build-system] build-backend = "setuptools.build_meta" requires = [ diff --git a/src/rapids_metadata/json.py b/src/rapids_metadata/json.py new file mode 100644 index 0000000..e96c40c --- /dev/null +++ b/src/rapids_metadata/json.py @@ -0,0 +1,90 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import dataclasses +import json +import os +import sys + +from . import rapids_metadata +from .metadata import ( + RAPIDSMetadata, + RAPIDSPackage, + RAPIDSRepository, + RAPIDSVersion, +) +from .rapids_version import get_rapids_version + + +__all__ = [ + "RAPIDSMetadataEncoder", + "main", +] + + +class RAPIDSMetadataEncoder(json.JSONEncoder): + def default(self, o): + def recurse(o): + if o is None: + return None + for t in [bool, int, float, str]: + if isinstance(o, t): + return o + if isinstance(o, dict): + return {key: recurse(value) for key, value in o.items()} + return self.default(o) + + for c in [RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository]: + if isinstance(o, c): + return { + field.name: recurse(getattr(o, field.name)) + for field in dataclasses.fields(o) + } + if isinstance(o, RAPIDSVersion): + return { + "repositories": { + str(repository): recurse(repository_data) + for repository, repository_data in o.repositories.items() + }, + **{ + field.name: recurse(getattr(o, field.name)) + for field in dataclasses.fields(o) + if field.name != "repositories" + }, + } + return super().default(o) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--all-versions", action="store_true") + + parsed = parser.parse_args() + metadata = ( + rapids_metadata + if parsed.all_versions + else RAPIDSMetadata( + versions={ + get_rapids_version(os.getcwd()): rapids_metadata.get_current_version( + os.getcwd() + ) + } + ) + ) + json.dump(metadata, sys.stdout, cls=RAPIDSMetadataEncoder) + + +if __name__ == "__main__": + main() diff --git a/tests/metadata/test_json.py b/tests/metadata/test_json.py new file mode 100644 index 0000000..da9b8dc --- /dev/null +++ b/tests/metadata/test_json.py @@ -0,0 +1,248 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import json +import os.path +from typing import Generator +from unittest.mock import patch + +import pytest +from rapids_metadata import json as rapids_json +from rapids_metadata.metadata import ( + PseudoRepository, + RAPIDSMetadata, + RAPIDSPackage, + RAPIDSRepository, + RAPIDSVersion, +) + + +@contextlib.contextmanager +def set_cwd(cwd: os.PathLike) -> Generator: + old_cwd = os.getcwd() + os.chdir(cwd) + try: + yield + finally: + os.chdir(old_cwd) + + +@pytest.mark.parametrize( + ["unencoded", "encoded"], + [ + (RAPIDSPackage(), {"has_alpha_spec": True, "has_cuda_suffix": True}), + ( + RAPIDSRepository( + packages={ + "package1": RAPIDSPackage(), + "package2": RAPIDSPackage( + has_alpha_spec=False, has_cuda_suffix=False + ), + } + ), + { + "packages": { + "package1": { + "has_alpha_spec": True, + "has_cuda_suffix": True, + }, + "package2": { + "has_alpha_spec": False, + "has_cuda_suffix": False, + }, + }, + }, + ), + ( + RAPIDSVersion( + repositories={ + "repo1": RAPIDSRepository(), + "repo2": RAPIDSRepository( + packages={ + "package": RAPIDSPackage(), + } + ), + PseudoRepository.NVIDIA: RAPIDSRepository( + packages={ + "proprietary-package": RAPIDSPackage(), + } + ), + } + ), + { + "repositories": { + "repo1": { + "packages": {}, + }, + "repo2": { + "packages": { + "package": { + "has_alpha_spec": True, + "has_cuda_suffix": True, + }, + }, + }, + "_nvidia": { + "packages": { + "proprietary-package": { + "has_alpha_spec": True, + "has_cuda_suffix": True, + }, + }, + }, + }, + }, + ), + ( + RAPIDSMetadata( + versions={ + "24.06": RAPIDSVersion(), + "24.08": RAPIDSVersion( + repositories={ + "repo": RAPIDSRepository(), + }, + ), + } + ), + { + "versions": { + "24.06": { + "repositories": {}, + }, + "24.08": { + "repositories": { + "repo": { + "packages": {}, + }, + }, + }, + }, + }, + ), + ], +) +def test_metadata_encoder(unencoded, encoded): + assert rapids_json.RAPIDSMetadataEncoder().default(unencoded) == encoded + + +@pytest.mark.parametrize( + ["version", "args", "expected_json"], + [ + ( + "24.08.00", + [], + { + "versions": { + "24.08": { + "repositories": { + "repo1": { + "packages": { + "package": { + "has_cuda_suffix": True, + "has_alpha_spec": True, + }, + }, + }, + }, + }, + }, + }, + ), + ( + "24.10.00", + [], + { + "versions": { + "24.10": { + "repositories": { + "repo2": { + "packages": { + "package": { + "has_cuda_suffix": True, + "has_alpha_spec": True, + }, + }, + }, + }, + }, + }, + }, + ), + ( + None, + ["--all-versions"], + { + "versions": { + "24.08": { + "repositories": { + "repo1": { + "packages": { + "package": { + "has_cuda_suffix": True, + "has_alpha_spec": True, + }, + }, + }, + }, + }, + "24.10": { + "repositories": { + "repo2": { + "packages": { + "package": { + "has_cuda_suffix": True, + "has_alpha_spec": True, + }, + }, + }, + }, + }, + }, + }, + ), + ], +) +def test_main(capsys, tmp_path, version, args, expected_json): + mock_metadata = RAPIDSMetadata( + versions={ + "24.08": RAPIDSVersion( + repositories={ + "repo1": RAPIDSRepository( + packages={ + "package": RAPIDSPackage(), + }, + ), + }, + ), + "24.10": RAPIDSVersion( + repositories={ + "repo2": RAPIDSRepository( + packages={ + "package": RAPIDSPackage(), + }, + ), + }, + ), + }, + ) + if version is not None: + with open(os.path.join(tmp_path, "VERSION"), "w") as f: + f.write(f"{version}\n") + with set_cwd(tmp_path), patch("sys.argv", ["rapids-metadata-json", *args]), patch( + "rapids_metadata.json.rapids_metadata", mock_metadata + ): + rapids_json.main() + captured = capsys.readouterr() + assert json.loads(captured.out) == expected_json From 028ea1c4a8aef01a48e8f61c8418f3dee6f52abb Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Jun 2024 09:57:26 -0400 Subject: [PATCH 2/7] Add test case --- tests/metadata/test_json.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/metadata/test_json.py b/tests/metadata/test_json.py index da9b8dc..ceaa39f 100644 --- a/tests/metadata/test_json.py +++ b/tests/metadata/test_json.py @@ -180,6 +180,26 @@ def test_metadata_encoder(unencoded, encoded): }, }, ), + ( + "24.12.00", + [], + { + "versions": { + "24.12": { + "repositories": { + "repo2": { + "packages": { + "package": { + "has_cuda_suffix": True, + "has_alpha_spec": True, + }, + }, + }, + }, + }, + }, + }, + ), ( None, ["--all-versions"], From 1c1bf6d52ad813176a596fa53055a16e75d21ee6 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Jun 2024 10:16:17 -0400 Subject: [PATCH 3/7] Add type hints --- src/rapids_metadata/json.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rapids_metadata/json.py b/src/rapids_metadata/json.py index e96c40c..8c25440 100644 --- a/src/rapids_metadata/json.py +++ b/src/rapids_metadata/json.py @@ -17,6 +17,7 @@ import json import os import sys +from typing import Any, Union from . import rapids_metadata from .metadata import ( @@ -35,7 +36,9 @@ class RAPIDSMetadataEncoder(json.JSONEncoder): - def default(self, o): + def default( + self, o: Union[RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository, RAPIDSVersion] + ) -> dict[str, Any]: def recurse(o): if o is None: return None From 9973fad53da9f3d850767ba077fe7dd39e0352c2 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Jun 2024 10:59:10 -0400 Subject: [PATCH 4/7] Remove PseudoRepository enum and simplify encoding --- src/rapids_metadata/__init__.py | 3 +-- src/rapids_metadata/json.py | 29 ++--------------------------- src/rapids_metadata/metadata.py | 14 +------------- tests/metadata/test_json.py | 3 +-- 4 files changed, 5 insertions(+), 44 deletions(-) diff --git a/src/rapids_metadata/__init__.py b/src/rapids_metadata/__init__.py index 866be61..0995e7e 100644 --- a/src/rapids_metadata/__init__.py +++ b/src/rapids_metadata/__init__.py @@ -13,7 +13,6 @@ # limitations under the License. from .metadata import ( - PseudoRepository, RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository, @@ -28,7 +27,7 @@ rapids_metadata.versions["24.08"] = RAPIDSVersion( repositories={ - PseudoRepository.NVIDIA: RAPIDSRepository( + "_nvidia": RAPIDSRepository( packages={ "cubinlinker": RAPIDSPackage(), } diff --git a/src/rapids_metadata/json.py b/src/rapids_metadata/json.py index 8c25440..51bb45a 100644 --- a/src/rapids_metadata/json.py +++ b/src/rapids_metadata/json.py @@ -39,34 +39,9 @@ class RAPIDSMetadataEncoder(json.JSONEncoder): def default( self, o: Union[RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository, RAPIDSVersion] ) -> dict[str, Any]: - def recurse(o): - if o is None: - return None - for t in [bool, int, float, str]: - if isinstance(o, t): - return o - if isinstance(o, dict): - return {key: recurse(value) for key, value in o.items()} - return self.default(o) - - for c in [RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository]: + for c in [RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository, RAPIDSVersion]: if isinstance(o, c): - return { - field.name: recurse(getattr(o, field.name)) - for field in dataclasses.fields(o) - } - if isinstance(o, RAPIDSVersion): - return { - "repositories": { - str(repository): recurse(repository_data) - for repository, repository_data in o.repositories.items() - }, - **{ - field.name: recurse(getattr(o, field.name)) - for field in dataclasses.fields(o) - if field.name != "repositories" - }, - } + return dataclasses.asdict(o) return super().default(o) diff --git a/src/rapids_metadata/metadata.py b/src/rapids_metadata/metadata.py index 65e705b..cd5bb4b 100644 --- a/src/rapids_metadata/metadata.py +++ b/src/rapids_metadata/metadata.py @@ -13,15 +13,12 @@ # limitations under the License. from dataclasses import dataclass, field -from enum import Enum from os import PathLike -from typing import Union from packaging.version import Version from .rapids_version import get_rapids_version __all__ = [ - "PseudoRepository", "RAPIDSMetadata", "RAPIDSPackage", "RAPIDSRepository", @@ -29,13 +26,6 @@ ] -class PseudoRepository(Enum): - NVIDIA = "nvidia" - - def __str__(self): - return f"_{self.value}" - - @dataclass class RAPIDSPackage: has_alpha_spec: bool = field(default=True) @@ -49,9 +39,7 @@ class RAPIDSRepository: @dataclass class RAPIDSVersion: - repositories: dict[Union[str, PseudoRepository], RAPIDSRepository] = field( - default_factory=dict - ) + repositories: dict[str, RAPIDSRepository] = field(default_factory=dict) @property def all_packages(self) -> set[str]: diff --git a/tests/metadata/test_json.py b/tests/metadata/test_json.py index ceaa39f..3e8344f 100644 --- a/tests/metadata/test_json.py +++ b/tests/metadata/test_json.py @@ -21,7 +21,6 @@ import pytest from rapids_metadata import json as rapids_json from rapids_metadata.metadata import ( - PseudoRepository, RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository, @@ -74,7 +73,7 @@ def set_cwd(cwd: os.PathLike) -> Generator: "package": RAPIDSPackage(), } ), - PseudoRepository.NVIDIA: RAPIDSRepository( + "_nvidia": RAPIDSRepository( packages={ "proprietary-package": RAPIDSPackage(), } From 09028071d3fbcd5e565e60fbcda209706ec2f873 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Jun 2024 11:06:03 -0400 Subject: [PATCH 5/7] Update README --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 66b2aea..21a9d55 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,21 @@ The motivating use case for this project is [`pre-commit-hooks`](https://github.com/rapidsai/pre-commit-hooks), but other projects may certainly use it too. +This package can also output the metadata in JSON form for consumption by +external programs, such as shell scripts and `jq`. To get JSON output, run: + +``` +rapids-metadata-json +``` + +This will print the metadata for the RAPIDS version specified by the `VERSION` +file in the current directory or above. If you wish to get metadata for all +RAPIDS versions instead, run: + +``` +rapids-metadata-json --all-versions +``` + ## Justification `pre-commit-hooks` has to know things about the structure of the RAPIDS project From 2033821a0a5ba5c89d8221d16c5c2f45ef8ab2ad Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Jun 2024 15:59:15 -0400 Subject: [PATCH 6/7] Review feedback --- src/rapids_metadata/json.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/rapids_metadata/json.py b/src/rapids_metadata/json.py index 51bb45a..877d2a5 100644 --- a/src/rapids_metadata/json.py +++ b/src/rapids_metadata/json.py @@ -39,10 +39,7 @@ class RAPIDSMetadataEncoder(json.JSONEncoder): def default( self, o: Union[RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository, RAPIDSVersion] ) -> dict[str, Any]: - for c in [RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository, RAPIDSVersion]: - if isinstance(o, c): - return dataclasses.asdict(o) - return super().default(o) + return dataclasses.asdict(o) def main(): From d5a66e02939bd5f06630236643434f42b015fa33 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 11 Jun 2024 16:11:50 -0400 Subject: [PATCH 7/7] Hide encoder class --- src/rapids_metadata/json.py | 5 ++--- tests/metadata/test_json.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/rapids_metadata/json.py b/src/rapids_metadata/json.py index 877d2a5..81864ef 100644 --- a/src/rapids_metadata/json.py +++ b/src/rapids_metadata/json.py @@ -30,12 +30,11 @@ __all__ = [ - "RAPIDSMetadataEncoder", "main", ] -class RAPIDSMetadataEncoder(json.JSONEncoder): +class _RAPIDSMetadataEncoder(json.JSONEncoder): def default( self, o: Union[RAPIDSMetadata, RAPIDSPackage, RAPIDSRepository, RAPIDSVersion] ) -> dict[str, Any]: @@ -58,7 +57,7 @@ def main(): } ) ) - json.dump(metadata, sys.stdout, cls=RAPIDSMetadataEncoder) + json.dump(metadata, sys.stdout, cls=_RAPIDSMetadataEncoder) if __name__ == "__main__": diff --git a/tests/metadata/test_json.py b/tests/metadata/test_json.py index 3e8344f..837d655 100644 --- a/tests/metadata/test_json.py +++ b/tests/metadata/test_json.py @@ -133,7 +133,7 @@ def set_cwd(cwd: os.PathLike) -> Generator: ], ) def test_metadata_encoder(unencoded, encoded): - assert rapids_json.RAPIDSMetadataEncoder().default(unencoded) == encoded + assert rapids_json._RAPIDSMetadataEncoder().default(unencoded) == encoded @pytest.mark.parametrize(