diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f6dcf8f..3422fe6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch ### New - adding universal environment replace in manifests (ref: `${SOMEKEY}`) - adding list recursion of manifests for environment variable replace +- adding support for AWS SecretsManager integration for pypi mirrors ### Changes - enforce strict validation for unknown values in manifests diff --git a/docs/source/index.rst b/docs/source/index.rst index 945fed77..721d2ee7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,6 +18,7 @@ It has a CommandLine Interface (CLI) based in Python. quick_start architecture installation + upgrades bootstrapping project_development manifests diff --git a/docs/source/manifests.md b/docs/source/manifests.md index 605ab7d7..20a19afe 100644 --- a/docs/source/manifests.md +++ b/docs/source/manifests.md @@ -31,6 +31,7 @@ targetAccountMappings: codebuildImage: XXXXXXXXXXXX.dkr.ecr.us-east-1.amazonaws.com/aws-codeseeder/code-build-base:5.5.0 npmMirror: https://registry.npmjs.org/ pypiMirror: https://pypi.python.org/simple + pypiMirrorSecret: /something/aws-addf-mirror-secret parametersGlobal: dockerCredentialsSecret: nameofsecret permissionsBoundaryName: policyname @@ -40,6 +41,7 @@ targetAccountMappings: codebuildImage: XXXXXXXXXXXX.dkr.ecr.us-east-1.amazonaws.com/aws-codeseeder/code-build-base:4.4.0 npmMirror: https://registry.npmjs.org/ pypiMirror: https://pypi.python.org/simple + pypiMirrorSecret: /something/aws-addf-mirror-secret parametersRegional: dockerCredentialsSecret: nameofsecret permissionsBoundaryName: policyname @@ -98,6 +100,7 @@ targetAccountMappings: - **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride)) - **npmMirror** - the NPM registry mirror to use (see [Mirror Override](mirroroverride)) - **pypiMirror** - the Pypi mirror to use (see [Mirror Override](mirroroverride)) + - **pypiMirrorSecret** - the AWS SecretManager to use when setting the mirror (see [Mirror Override](mirroroverride)) - **parametersGlobal** - these are parameters that apply to all region mappings unless otherwise overridden at the region level - **dockerCredentialsSecret** - please see [Docker Credentials Secret](dockerCredentialsSecret) - **permissionsBoundaryName** - the name of the [permissions boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) policy to apply to all module-specific roles created @@ -107,6 +110,7 @@ targetAccountMappings: - **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride)) - **npmMirror** - the NPM registry mirror to use (see [Mirror Override](mirroroverride)) - **pypiMirror** - the Pypi mirror to use (see [Mirror Override](mirroroverride)) + - **pypiMirrorSecret** - the AWS SecretManager to use when setting the mirror (see [Mirror Override](mirroroverride)) - **parametersRegional** - these are parameters that apply to all region mappings unless otherwise overridden at the region level - **dockerCredentialsSecret** - please see [Docker Credentials Secret](dockerCredentialsSecret) - This is a NAMED PARAMETER...in that `dockerCredentialsSecret` is recognized by `seed-farmer` @@ -251,6 +255,7 @@ targetRegion: us-west-2 codebuildImage: XXXXXXXXXXXX.dkr.ecr.us-east-1.amazonaws.com/aws-codeseeder/code-build-base:3.3.0 npmMirror: https://registry.npmjs.org/ pypiMirror: https://pypi.python.org/simple +pypiMirrorSecret: /something/aws-addf-mirror-secret parameters: - name: encryption-type value: SSE @@ -275,6 +280,7 @@ dataFiles: - **codebuildImage** - a custom build image to use (see [Build Image Override](buildimageoverride)) - **npmMirror** - the NPM registry mirror to use (see [Mirror Override](mirroroverride)) - **pypiMirror** - the Pypi mirror to use (see [Mirror Override](mirroroverride)) +- **pypiMirrorSecret** - the AWS SecretManager to use when setting the mirror (see [Mirror Override](mirroroverride)) - **parameters** - the parameters section .... see [Parameters](parameters) - **dataFiles** - additional files to add to the bundle that are outside of the module code - this is LIST and EVERY element in the list must have the keyword **filePath** @@ -389,6 +395,75 @@ There are three (3) places to configure a custom build image: 3. if a mirror is defined at the account level --- USE IT... ELSE 4. no mirror is set +### Mirror Secrets +If using a Pypi-compliant mirror that is not public or needs an authentication scheme, the `pypiMirrorSecret` provides a means to set a username / password (or user / token) in the global definition of the AWS Codebuild runtime. To use this feature, you MUST adhere to the following: +1. Be sure to have `seed-farmer` version >= 3.5.0 and have properly updated if migrating from an older version +2. have an AWS SecretsManager set in EACH account/region combination that you want to leverage it (the user MUST set this up prior to using) +3. the content of the AWS SecretsManager adheres to the format defined below +4. the name of the AWS SecretsManager adheres to the format defined below +5. a `pypiMirror` is defined at the SAME LEVEL in the manifest + +The `pypiMirrorSecret` feature can support multiple entries for use. It is NOT apart of the calculation for redeploy (ie. you can change it at will and it will not force a redeploy of any module referencing it - to allow updates and additions over time). + +The AWS SecretManager name must follow the following pattern (NO exceptions): +```code +*-mirror-credentials* +``` +Here are some examples of valid names: +- /aws-addf-mirror-credentials +- /something/important/hey-mirror-credentials + +Here are some names that are in-valid +- /aws-addfmirror-credentials +- /something/important/mirror-credentials + +The content of the AWS SecretsManager allows multiple entries to support different configuratons. It MUST be a JSON dict of dict, where each top-level dict is the name of the key-par and its child elements contain `username` and `password` keys. Lets look at an example of a value payload: + +```json +{ + "pypi": { + "username": "derekpypi", + "password": "thepasswordpypi" + }, + "artifactory": { + "username": "myuser@amazon.com", + "password": "agobbleygookofahexcodehere" + }, + "pypi2": { + "username": "hey", + "password": "yooooo" + }, +} +``` +This example has valid entries. The default key is `pypi`. In order to leverage this scheme, a particular pattern MUST be followed in your manifest under the `pypiMirrorSecret` key: `name-of-secret::name-of-key`. The `::` indicates to `seed-farmer` and `AWS-CodeSeeder` what username/password combination to use. + +Lets walk thru an example. Assume that the previous example of a secret payload is set and the secret name is `/aws-addf-mirror-credentials`. I want to use the `artifactory` username/password entry. My manifest would look like the following: +```yaml +... +pypiMirror: https://the-mirror-dns/simple/pypi +pypiMirrorSecret: /aws-addf-mirror-credentials::artifactory +... + +``` +This would result in the creation of the url `https://myuser@amazon.com:agobbleygookofahexcodehere@the-mirror-dns/simple/pypi` and the global config in the runtime will be set via: + +```code +pip config set global.index-url https://myuser@amazon.com:agobbleygookofahexcodehere@the-mirror-dns/simple/pypi + +``` + +If I wanted to user the default `pypi` entry, my manifest would look like: +```yaml +... +pypiMirror: https://the-mirror-dns/simple/pypi +pypiMirrorSecret: /aws-addf-mirror-credentials +... + +``` +This would result in the creation of the url `https://derekpypi:thepasswordpypi@the-mirror-dns/simple/pypi` and the global config in the runtime will be set via: +```code +pip config set global.index-url https://derekpypi:thepasswordpypi@the-mirror-dns/simple/pypi +``` diff --git a/docs/source/upgrades.md b/docs/source/upgrades.md new file mode 100644 index 00000000..0f150edd --- /dev/null +++ b/docs/source/upgrades.md @@ -0,0 +1,17 @@ +# Upgrades + +This page give indications related to supporting upgrading `seed-farmer` from previous version where changes must be enacted in order to properly use new feature. + +This is not an exhaustive list but merely a guide. + + +## Upgrading to 3.5.0 + +`seed-farmer` 3.5.0 introduces the use of `pypiMirrorSecret` to support configuring a pypi mirror with credentials (see [Manifests - Mirrors](./manifests.md#mirroroverride)). + +If you are currently using an older version of `seed-farmer`, to upgrade you must install the 3.5.0 version from Pypi and then update your current deployments by updating the seedkit in each target account/region mapping via the CLI using the `--update-seedkit` flag. This is needed only once +```code +seedfarmer apply --update-seedkit +``` + +Please see the [CLI references](./cli_commands.rst) for more details \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 05417cca..11e7c8e9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ annotated-types==0.5.0 # via pydantic arrow==1.2.3 # via jinja2-time -aws-codeseeder==0.12.0 +aws-codeseeder==0.13.0 # via seed-farmer (setup.py) binaryornot==0.4.4 # via cookiecutter diff --git a/seedfarmer/commands/_module_commands.py b/seedfarmer/commands/_module_commands.py index ad57803c..bb5ec54b 100644 --- a/seedfarmer/commands/_module_commands.py +++ b/seedfarmer/commands/_module_commands.py @@ -54,6 +54,7 @@ def _env_vars( permissions_boundary_arn: Optional[str] = None, session: Optional[Session] = None, use_project_prefix: Optional[bool] = True, + pypi_mirror_secret: Optional[str] = None, ) -> Dict[str, str]: env_vars = ( { @@ -76,6 +77,8 @@ def _env_vars( env_vars["AWS_CODESEEDER_DOCKER_SECRET"] = docker_credentials_secret if permissions_boundary_arn: env_vars[_param("PERMISSIONS_BOUNDARY_ARN", use_project_prefix)] = permissions_boundary_arn + if pypi_mirror_secret is not None: + env_vars["AWS_CODESEEDER_MIRROR_SECRET"] = pypi_mirror_secret # Add the partition to env for ease of fetching env_vars["AWS_PARTITION"] = deployment_partition env_vars["AWS_CODESEEDER_VERSION"] = aws_codeseeder.__version__ @@ -102,6 +105,9 @@ def deploy_module(mdo: ModuleDeployObject) -> ModuleDeploymentResponse: permissions_boundary_arn=mdo.permissions_boundary_arn, session=SessionManager().get_or_create().get_deployment_session(account_id=account_id, region_name=region), use_project_prefix=use_project_prefix, + pypi_mirror_secret=( + module_manifest.pypi_mirror_secret if module_manifest.pypi_mirror_secret else mdo.pypi_mirror_secret + ), ) env_vars[_param("MODULE_MD5", use_project_prefix)] = ( module_manifest.bundle_md5 if module_manifest.bundle_md5 is not None else "" @@ -213,6 +219,9 @@ def destroy_module(mdo: ModuleDeployObject) -> ModuleDeploymentResponse: module_metadata=mdo.module_metadata, session=SessionManager().get_or_create().get_deployment_session(account_id=account_id, region_name=region), use_project_prefix=use_project_prefix, + pypi_mirror_secret=( + module_manifest.pypi_mirror_secret if module_manifest.pypi_mirror_secret else mdo.pypi_mirror_secret + ), ) remove_ssm = [ diff --git a/seedfarmer/models/manifests/_deployment_manifest.py b/seedfarmer/models/manifests/_deployment_manifest.py index ef29bb81..ce6149e2 100644 --- a/seedfarmer/models/manifests/_deployment_manifest.py +++ b/seedfarmer/models/manifests/_deployment_manifest.py @@ -59,6 +59,7 @@ class RegionMapping(CamelModel): codebuild_image: Optional[str] = None npm_mirror: Optional[str] = None pypi_mirror: Optional[str] = None + pypi_mirror_secret: Optional[str] = None seedkit_metadata: Optional[Dict[str, Any]] = None @@ -76,6 +77,7 @@ class TargetAccountMapping(CamelModel): codebuild_image: Optional[str] = None npm_mirror: Optional[str] = None pypi_mirror: Optional[str] = None + pypi_mirror_secret: Optional[str] = None _default_region: Optional[RegionMapping] = PrivateAttr(default=None) _region_index: Dict[str, RegionMapping] = PrivateAttr(default_factory=dict) @@ -368,6 +370,36 @@ def get_region_pypi_mirror( else: return None + def get_region_pypi_mirror_secret( + self, + *, + account_alias: Optional[str] = None, + account_id: Optional[str] = None, + region: Optional[str] = None, + ) -> Optional[str]: + if account_alias is not None and account_id is not None: + raise seedfarmer.errors.InvalidManifestError("Only one of 'account_alias' and 'account_id' is allowed") + + use_default_account = account_alias is None and account_id is None + use_default_region = region is None + for target_account in self.target_account_mappings: + if ( + account_alias == target_account.alias + or account_id == target_account.actual_account_id + or (use_default_account and target_account.default) + ): + # Search the region_mappings for the region, if the pypi_mirror_secret is in region + for region_mapping in target_account.region_mappings: + if region == region_mapping.region or (use_default_region and region_mapping.default): + pypi_mirror_secret = ( + region_mapping.pypi_mirror_secret + if region_mapping.pypi_mirror_secret is not None + else target_account.pypi_mirror_secret + ) + return pypi_mirror_secret + else: + return None + def validate_and_set_module_defaults(self) -> None: for group in self.groups: for module in group.modules: diff --git a/seedfarmer/models/manifests/_module_manifest.py b/seedfarmer/models/manifests/_module_manifest.py index 0e3db156..8804fe8b 100644 --- a/seedfarmer/models/manifests/_module_manifest.py +++ b/seedfarmer/models/manifests/_module_manifest.py @@ -98,6 +98,7 @@ class ModuleManifest(CamelModel): commit_hash: SkipJsonSchema[Optional[str]] = None npm_mirror: Optional[str] = None pypi_mirror: Optional[str] = None + pypi_mirror_secret: Optional[str] = None _target_account_id: Optional[str] = PrivateAttr(default=None) _local_path: Optional[str] = PrivateAttr(default=None) diff --git a/seedfarmer/models/transfer/_module_deploy_object.py b/seedfarmer/models/transfer/_module_deploy_object.py index c653ed4f..97facd92 100644 --- a/seedfarmer/models/transfer/_module_deploy_object.py +++ b/seedfarmer/models/transfer/_module_deploy_object.py @@ -17,6 +17,7 @@ class ModuleDeployObject(CamelModel): codebuild_image: Optional[str] = None npm_mirror: Optional[str] = None pypi_mirror: Optional[str] = None + pypi_mirror_secret: Optional[str] = None def _render_permissions_boundary_arn( self, account_id: Optional[str], partition: Optional[str], permissions_boundary_name: Optional[str] @@ -54,8 +55,13 @@ def __init__(self, **kwargs: Any) -> None: account_alias=_module.target_account, region=_module.target_region ) + pypi_mirror_secret = self.deployment_manifest.get_region_pypi_mirror_secret( + account_alias=_module.target_account, region=_module.target_region + ) + self.permissions_boundary_arn = pba if pba is not None else None self.codebuild_image = codebuild_image if codebuild_image is not None else None self.docker_credentials_secret = dcs if dcs else None self.npm_mirror = npm_mirror if npm_mirror is not None else None self.pypi_mirror = pypi_mirror if pypi_mirror is not None else None + self.pypi_mirror_secret = pypi_mirror_secret if pypi_mirror_secret is not None else None diff --git a/setup.py b/setup.py index e47207bb..2ffeeed8 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ keywords=["aws", "cdk"], python_requires=">=3.8,<3.12", install_requires=[ - "aws-codeseeder~=0.12.0", + "aws-codeseeder~=0.13.0", "cookiecutter~=2.1.0", "pyhumps~=3.5.0", "pydantic~=2.5.3",