diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 000000000..2ef50d3ca --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,114 @@ +# Copyright 2024 The Sigstore Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pathlib import Path + +import pytest +from id import ( + AmbientCredentialError, + GitHubOidcPermissionCredentialError, + detect_credential, +) + +from sigstore.oidc import _DEFAULT_AUDIENCE + +_ASSETS = (Path(__file__).parent / "assets").resolve() +assert _ASSETS.is_dir() + + +@pytest.fixture +def asset(): + def _asset(name: str) -> Path: + return _ASSETS / name + + return _asset + + +def _has_oidc_id(): + # If there are tokens manually defined for us in the environment, use them. + if os.getenv("SIGSTORE_IDENTITY_TOKEN_production") or os.getenv( + "SIGSTORE_IDENTITY_TOKEN_staging" + ): + return True + + try: + token = detect_credential(_DEFAULT_AUDIENCE) + if token is None: + return False + except GitHubOidcPermissionCredentialError: + # On GitHub Actions, forks do not have access to OIDC identities. + # We differentiate this case from other GitHub credential errors, + # since it's a case where we want to skip (i.e. return False). + if os.getenv("GITHUB_EVENT_NAME") == "pull_request": + return False + return True + except AmbientCredentialError: + # If ambient credential detection raises, then we *are* in an ambient + # environment but one that's been configured incorrectly. We + # pass this through, so that the CI fails appropriately rather than + # silently skipping the faulty tests. + return True + + return True + + +def pytest_addoption(parser): + parser.addoption( + "--skip-online", + action="store_true", + help="skip tests that require network connectivity", + ) + parser.addoption( + "--skip-staging", + action="store_true", + help="skip tests that require Sigstore staging infrastructure", + ) + + +def pytest_runtest_setup(item): + # Do we need a network connection? + online = False + for mark in ["online", "staging", "production"]: + if mark in item.keywords: + online = True + + if online and item.config.getoption("--skip-online"): + pytest.skip( + "skipping test that requires network connectivity due to `--skip-online` flag" + ) + elif "ambient_oidc" in item.keywords and not _has_oidc_id(): + pytest.skip("skipping test that requires an ambient OIDC credential") + + if "staging" in item.keywords and item.config.getoption("--skip-staging"): + pytest.skip( + "skipping test that requires staging infrastructure due to `--skip-staging` flag" + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "staging: mark test as requiring Sigstore staging infrastructure" + ) + config.addinivalue_line( + "markers", + "production: mark test as requiring Sigstore production infrastructure", + ) + config.addinivalue_line( + "markers", + "online: mark test as requiring network connectivity (but not a specific Sigstore infrastructure)", + ) + config.addinivalue_line( + "markers", "ambient_oidc: mark test as requiring an ambient OIDC identity" + ) diff --git a/test/integration/cli/conftest.py b/test/integration/cli/conftest.py index 282795350..abd2b9cc8 100644 --- a/test/integration/cli/conftest.py +++ b/test/integration/cli/conftest.py @@ -11,94 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import os from pathlib import Path from typing import Callable import pytest -from id import ( - AmbientCredentialError, - GitHubOidcPermissionCredentialError, - detect_credential, -) from sigstore._cli import main -from sigstore.oidc import _DEFAULT_AUDIENCE - -_ASSETS = (Path(__file__).parent.parent.parent / "assets/integration").resolve() -assert _ASSETS.is_dir() - - -def _has_oidc_id(): - # If there are tokens manually defined for us in the environment, use them. - if os.getenv("SIGSTORE_IDENTITY_TOKEN_production") or os.getenv( - "SIGSTORE_IDENTITY_TOKEN_staging" - ): - return True - - try: - token = detect_credential(_DEFAULT_AUDIENCE) - if token is None: - return False - except GitHubOidcPermissionCredentialError: - # On GitHub Actions, forks do not have access to OIDC identities. - # We differentiate this case from other GitHub credential errors, - # since it's a case where we want to skip (i.e. return False). - if os.getenv("GITHUB_EVENT_NAME") == "pull_request": - return False - return True - except AmbientCredentialError: - # If ambient credential detection raises, then we *are* in an ambient - # environment but one that's been configured incorrectly. We - # pass this through, so that the CI fails appropriately rather than - # silently skipping the faulty tests. - return True - - return True - - -def pytest_runtest_setup(item): - # Do we need a network connection? - online = False - for mark in ["online", "staging", "production"]: - if mark in item.keywords: - online = True - - if online and item.config.getoption("--skip-online"): - pytest.skip( - "skipping test that requires network connectivity due to `--skip-online` flag" - ) - elif "ambient_oidc" in item.keywords and not _has_oidc_id(): - pytest.skip("skipping test that requires an ambient OIDC credential") - - if "staging" in item.keywords and item.config.getoption("--skip-staging"): - pytest.skip( - "skipping test that requires staging infrastructure due to `--skip-staging` flag" - ) - - -def pytest_configure(config): - config.addinivalue_line( - "markers", "staging: mark test as requiring Sigstore staging infrastructure" - ) - config.addinivalue_line( - "markers", - "production: mark test as requiring Sigstore production infrastructure", - ) - config.addinivalue_line( - "markers", - "online: mark test as requiring network connectivity (but not a specific Sigstore infrastructure)", - ) - config.addinivalue_line( - "markers", "ambient_oidc: mark test as requiring an ambient OIDC identity" - ) @pytest.fixture -def asset(): +def asset_integration(asset): def _asset(name: str) -> Path: - return _ASSETS / name + return asset(f"integration/{name}") return _asset diff --git a/test/integration/cli/test_attest.py b/test/integration/cli/test_attest.py index 813823452..db41ee33a 100644 --- a/test/integration/cli/test_attest.py +++ b/test/integration/cli/test_attest.py @@ -56,10 +56,10 @@ def get_cli_params( ], ) def test_attest_success_default_output_bundle( - capsys, sigstore, asset, predicate_type, predicate_filename + capsys, sigstore, asset_integration, predicate_type, predicate_filename ): - predicate_path = asset(f"attest/{predicate_filename}") - artifact = asset("a.txt") + predicate_path = asset_integration(f"attest/{predicate_filename}") + artifact = asset_integration("a.txt") expected_output_bundle = artifact.with_name("a.txt.sigstore.json") assert not expected_output_bundle.exists() @@ -87,11 +87,13 @@ def test_attest_success_default_output_bundle( @pytest.mark.staging @pytest.mark.ambient_oidc -def test_attest_success_custom_output_bundle(capsys, sigstore, asset, tmp_path): +def test_attest_success_custom_output_bundle( + capsys, sigstore, asset_integration, tmp_path +): predicate_type = PredicateType.SLSA_v0_2 predicate_filename = "slsa_predicate_v0_2.json" - predicate_path = asset(f"attest/{predicate_filename}") - artifact = asset("a.txt") + predicate_path = asset_integration(f"attest/{predicate_filename}") + artifact = asset_integration("a.txt") output_bundle = tmp_path / "bundle.json" assert not output_bundle.exists() @@ -111,11 +113,13 @@ def test_attest_success_custom_output_bundle(capsys, sigstore, asset, tmp_path): @pytest.mark.staging @pytest.mark.ambient_oidc -def test_attest_overwrite_existing_bundle(capsys, sigstore, asset, tmp_path): +def test_attest_overwrite_existing_bundle( + capsys, sigstore, asset_integration, tmp_path +): predicate_type = PredicateType.SLSA_v0_2 predicate_filename = "slsa_predicate_v0_2.json" - predicate_path = asset(f"attest/{predicate_filename}") - artifact = asset("a.txt") + predicate_path = asset_integration(f"attest/{predicate_filename}") + artifact = asset_integration("a.txt") output_bundle = tmp_path / "bundle.json" assert not output_bundle.exists() @@ -148,11 +152,11 @@ def test_attest_overwrite_existing_bundle(capsys, sigstore, asset, tmp_path): assert captures.out.endswith(f"Sigstore bundle written to {str(output_bundle)}\n") -def test_attest_invalid_predicate_type(capsys, sigstore, asset, tmp_path): +def test_attest_invalid_predicate_type(capsys, sigstore, asset_integration, tmp_path): predicate_type = "invalid_type" predicate_filename = "slsa_predicate_v0_2.json" - predicate_path = asset(f"attest/{predicate_filename}") - artifact = asset("a.txt") + predicate_path = asset_integration(f"attest/{predicate_filename}") + artifact = asset_integration("a.txt") output_bundle = tmp_path / "bundle.json" # On invalid argument errors we call `Argumentparser.error`, which prints @@ -172,11 +176,11 @@ def test_attest_invalid_predicate_type(capsys, sigstore, asset, tmp_path): assert captures.err.endswith(f"invalid PredicateType value: '{predicate_type}'\n") -def test_attest_mismatching_predicate(capsys, sigstore, asset, tmp_path): +def test_attest_mismatching_predicate(capsys, sigstore, asset_integration, tmp_path): predicate_type = PredicateType.SLSA_v0_2 predicate_filename = "slsa_predicate_v1_0.json" - predicate_path = asset(f"attest/{predicate_filename}") - artifact = asset("a.txt") + predicate_path = asset_integration(f"attest/{predicate_filename}") + artifact = asset_integration("a.txt") output_bundle = tmp_path / "bundle.json" # On invalid argument errors we call `Argumentparser.error`, which prints @@ -196,11 +200,11 @@ def test_attest_mismatching_predicate(capsys, sigstore, asset, tmp_path): assert f'Unable to parse predicate of type "{predicate_type}":' in captures.err -def test_attest_missing_predicate(capsys, sigstore, asset, tmp_path): +def test_attest_missing_predicate(capsys, sigstore, asset_integration, tmp_path): predicate_type = PredicateType.SLSA_v0_2 predicate_filename = "doesnt_exist.json" - predicate_path = asset(f"attest/{predicate_filename}") - artifact = asset("a.txt") + predicate_path = asset_integration(f"attest/{predicate_filename}") + artifact = asset_integration("a.txt") output_bundle = tmp_path / "bundle.json" # On invalid argument errors we call `Argumentparser.error`, which prints @@ -220,10 +224,10 @@ def test_attest_missing_predicate(capsys, sigstore, asset, tmp_path): assert captures.err.endswith(f"Predicate must be a file: {predicate_path}\n") -def test_attest_invalid_json_predicate(capsys, sigstore, asset, tmp_path): +def test_attest_invalid_json_predicate(capsys, sigstore, asset_integration, tmp_path): predicate_type = PredicateType.SLSA_v0_2 - predicate_path = asset("a.txt") - artifact = asset("a.txt") + predicate_path = asset_integration("a.txt") + artifact = asset_integration("a.txt") output_bundle = tmp_path / "bundle.json" # On invalid argument errors we call `Argumentparser.error`, which prints diff --git a/test/integration/cli/test_plumbing.py b/test/integration/cli/test_plumbing.py index ebcf8e7fc..62c014ded 100644 --- a/test/integration/cli/test_plumbing.py +++ b/test/integration/cli/test_plumbing.py @@ -22,8 +22,8 @@ from sigstore.verify.verifier import Verifier -def test_fix_bundle_fixes_missing_checkpoint(capsys, sigstore, asset): - invalid_bundle = asset("Python-3.12.5.tgz.sigstore") +def test_fix_bundle_fixes_missing_checkpoint(capsys, sigstore, asset_integration): + invalid_bundle = asset_integration("Python-3.12.5.tgz.sigstore") # The bundle is invalid, because it's missing a checkpoint # for its inclusion proof. @@ -64,8 +64,8 @@ def test_fix_bundle_fixes_missing_checkpoint(capsys, sigstore, asset): ) -def test_fix_bundle_upgrades_bundle(capsys, sigstore, asset): - invalid_bundle = asset("Python-3.12.5.tgz.sigstore") +def test_fix_bundle_upgrades_bundle(capsys, sigstore, asset_integration): + invalid_bundle = asset_integration("Python-3.12.5.tgz.sigstore") # Running `sigstore plumbing fix-bundle --upgrade-version` # emits a fixed bundle. diff --git a/test/integration/cli/test_sign.py b/test/integration/cli/test_sign.py index b666e807e..0cf1eb684 100644 --- a/test/integration/cli/test_sign.py +++ b/test/integration/cli/test_sign.py @@ -51,8 +51,8 @@ def get_cli_params( @pytest.mark.staging @pytest.mark.ambient_oidc -def test_sign_success_default_output_bundle(capsys, sigstore, asset): - artifact = asset("a.txt") +def test_sign_success_default_output_bundle(capsys, sigstore, asset_integration): + artifact = asset_integration("a.txt") expected_output_bundle = artifact.with_name("a.txt.sigstore.json") assert not expected_output_bundle.exists() @@ -82,8 +82,8 @@ def test_sign_success_default_output_bundle(capsys, sigstore, asset): @pytest.mark.staging @pytest.mark.ambient_oidc -def test_sign_success_custom_outputs(capsys, sigstore, asset, tmp_path): - artifact = asset("a.txt") +def test_sign_success_custom_outputs(capsys, sigstore, asset_integration, tmp_path): + artifact = asset_integration("a.txt") output_bundle = tmp_path / "bundle.json" output_cert = tmp_path / "cert.cert" output_signature = tmp_path / "signature.sig" @@ -109,8 +109,8 @@ def test_sign_success_custom_outputs(capsys, sigstore, asset, tmp_path): @pytest.mark.staging @pytest.mark.ambient_oidc -def test_sign_success_custom_output_dir(capsys, sigstore, asset, tmp_path): - artifact = asset("a.txt") +def test_sign_success_custom_output_dir(capsys, sigstore, asset_integration, tmp_path): + artifact = asset_integration("a.txt") expected_output_bundle = tmp_path / "a.txt.sigstore.json" sigstore( @@ -130,8 +130,8 @@ def test_sign_success_custom_output_dir(capsys, sigstore, asset, tmp_path): @pytest.mark.staging @pytest.mark.ambient_oidc -def test_sign_success_no_default_files(capsys, sigstore, asset, tmp_path): - artifact = asset("a.txt") +def test_sign_success_no_default_files(capsys, sigstore, asset_integration, tmp_path): + artifact = asset_integration("a.txt") default_output_bundle = tmp_path / "a.txt.sigstore.json" output_cert = tmp_path / "cert.cert" output_signature = tmp_path / "sig.sig" @@ -156,8 +156,8 @@ def test_sign_success_no_default_files(capsys, sigstore, asset, tmp_path): @pytest.mark.staging @pytest.mark.ambient_oidc -def test_sign_overwrite_existing_bundle(capsys, sigstore, asset): - artifact = asset("a.txt") +def test_sign_overwrite_existing_bundle(capsys, sigstore, asset_integration): + artifact = asset_integration("a.txt") expected_output_bundle = artifact.with_name("a.txt.sigstore.json") assert not expected_output_bundle.exists() @@ -194,8 +194,10 @@ def test_sign_overwrite_existing_bundle(capsys, sigstore, asset): expected_output_bundle.unlink() -def test_sign_fails_with_default_files_and_bundle_options(capsys, sigstore, asset): - artifact = asset("a.txt") +def test_sign_fails_with_default_files_and_bundle_options( + capsys, sigstore, asset_integration +): + artifact = asset_integration("a.txt") output_bundle = artifact.with_name("a.txt.sigstore.json") with pytest.raises(SystemExit) as e: @@ -214,8 +216,10 @@ def test_sign_fails_with_default_files_and_bundle_options(capsys, sigstore, asse ) -def test_sign_fails_with_multiple_inputs_and_custom_output(capsys, sigstore, asset): - artifact = asset("a.txt") +def test_sign_fails_with_multiple_inputs_and_custom_output( + capsys, sigstore, asset_integration +): + artifact = asset_integration("a.txt") with pytest.raises(SystemExit) as e: sigstore( @@ -257,8 +261,10 @@ def test_sign_fails_with_multiple_inputs_and_custom_output(capsys, sigstore, ass ) -def test_sign_fails_with_output_dir_and_custom_output_files(capsys, sigstore, asset): - artifact = asset("a.txt") +def test_sign_fails_with_output_dir_and_custom_output_files( + capsys, sigstore, asset_integration +): + artifact = asset_integration("a.txt") with pytest.raises(SystemExit) as e: sigstore( @@ -303,8 +309,10 @@ def test_sign_fails_with_output_dir_and_custom_output_files(capsys, sigstore, as ) -def test_sign_fails_without_both_output_cert_and_signature(capsys, sigstore, asset): - artifact = asset("a.txt") +def test_sign_fails_without_both_output_cert_and_signature( + capsys, sigstore, asset_integration +): + artifact = asset_integration("a.txt") with pytest.raises(SystemExit) as e: sigstore( diff --git a/test/unit/conftest.py b/test/unit/conftest.py index 2c55bf6a7..80a0893a2 100644 --- a/test/unit/conftest.py +++ b/test/unit/conftest.py @@ -27,8 +27,6 @@ import pytest from cryptography.x509 import Certificate, load_pem_x509_certificate from id import ( - AmbientCredentialError, - GitHubOidcPermissionCredentialError, detect_credential, ) from tuf.api.exceptions import DownloadHTTPError @@ -44,103 +42,14 @@ from sigstore.sign import SigningContext from sigstore.verify.verifier import Verifier -_ASSETS = (Path(__file__).parent.parent / "assets").resolve() -assert _ASSETS.is_dir() - -_TUF_ASSETS = (_ASSETS / "staging-tuf").resolve() +_TUF_ASSETS = (Path(__file__).parent.parent / "assets" / "staging-tuf").resolve() assert _TUF_ASSETS.is_dir() -def _has_oidc_id(): - # If there are tokens manually defined for us in the environment, use them. - if os.getenv("SIGSTORE_IDENTITY_TOKEN_production") or os.getenv( - "SIGSTORE_IDENTITY_TOKEN_staging" - ): - return True - - try: - token = detect_credential(_DEFAULT_AUDIENCE) - if token is None: - return False - except GitHubOidcPermissionCredentialError: - # On GitHub Actions, forks do not have access to OIDC identities. - # We differentiate this case from other GitHub credential errors, - # since it's a case where we want to skip (i.e. return False). - if os.getenv("GITHUB_EVENT_NAME") == "pull_request": - return False - return True - except AmbientCredentialError: - # If ambient credential detection raises, then we *are* in an ambient - # environment but one that's been configured incorrectly. We - # pass this through, so that the CI fails appropriately rather than - # silently skipping the faulty tests. - return True - - return True - - -def pytest_addoption(parser): - parser.addoption( - "--skip-online", - action="store_true", - help="skip tests that require network connectivity", - ) - parser.addoption( - "--skip-staging", - action="store_true", - help="skip tests that require Sigstore staging infrastructure", - ) - - -def pytest_runtest_setup(item): - # Do we need a network connection? - online = False - for mark in ["online", "staging", "production"]: - if mark in item.keywords: - online = True - - if online and item.config.getoption("--skip-online"): - pytest.skip( - "skipping test that requires network connectivity due to `--skip-online` flag" - ) - elif "ambient_oidc" in item.keywords and not _has_oidc_id(): - pytest.skip("skipping test that requires an ambient OIDC credential") - - if "staging" in item.keywords and item.config.getoption("--skip-staging"): - pytest.skip( - "skipping test that requires staging infrastructure due to `--skip-staging` flag" - ) - - -def pytest_configure(config): - config.addinivalue_line( - "markers", "staging: mark test as requiring Sigstore staging infrastructure" - ) - config.addinivalue_line( - "markers", - "production: mark test as requiring Sigstore production infrastructure", - ) - config.addinivalue_line( - "markers", - "online: mark test as requiring network connectivity (but not a specific Sigstore infrastructure)", - ) - config.addinivalue_line( - "markers", "ambient_oidc: mark test as requiring an ambient OIDC identity" - ) - - -@pytest.fixture -def asset(): - def _asset(name: str) -> Path: - return _ASSETS / name - - return _asset - - @pytest.fixture -def x509_testcase(): +def x509_testcase(asset): def _x509_testcase(name: str) -> Certificate: - pem = (_ASSETS / "x509" / name).read_bytes() + pem = asset(f"x509/{name}").read_bytes() return load_pem_x509_certificate(pem) return _x509_testcase @@ -179,13 +88,13 @@ def target_path(self, name: str) -> Path: @pytest.fixture -def signing_materials() -> Callable[[str, RekorClient], tuple[Path, Bundle]]: +def signing_materials(asset) -> Callable[[str, RekorClient], tuple[Path, Bundle]]: # NOTE: Unlike `signing_bundle`, `signing_materials` requires a # Rekor client to retrieve its entry with. def _signing_materials(name: str, client: RekorClient) -> tuple[Path, Bundle]: - file = _ASSETS / name - cert_path = _ASSETS / f"{name}.crt" - sig_path = _ASSETS / f"{name}.sig" + file = asset(name) + cert_path = asset(f"{name}.crt") + sig_path = asset(f"{name}.sig") cert = load_pem_x509_certificate(cert_path.read_bytes()) sig = base64.b64decode(sig_path.read_text()) @@ -204,10 +113,10 @@ def _signing_materials(name: str, client: RekorClient) -> tuple[Path, Bundle]: @pytest.fixture -def signing_bundle(): +def signing_bundle(asset): def _signing_bundle(name: str) -> tuple[Path, Bundle]: - file = _ASSETS / name - bundle_path = _ASSETS / f"{name}.sigstore" + file = asset(name) + bundle_path = asset(f"{name}.sigstore") bundle = Bundle.from_json(bundle_path.read_bytes()) return (file, bundle)