diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 37fa329f..ac4fe899 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,15 +15,16 @@ jobs: seedfarmer-cli: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: [3.9,3.10.13] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] defaults: run: working-directory: . steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install Requirements @@ -33,7 +34,7 @@ jobs: pip install -r requirements-dev.txt pip install -e . - name: Mypy Check - run: mypy --ignore-missing-imports ./seedfarmer + run: mypy ./seedfarmer - name: Flake8 Check run: flake8 ./seedfarmer - name: Black Check diff --git a/.gitignore b/.gitignore index 86536db3..7193d61e 100644 --- a/.gitignore +++ b/.gitignore @@ -115,7 +115,7 @@ web_modules/ # dotenv environment variables file .env -.env.test +.env.* # parcel-bundler cache (https://parceljs.org/) .cache @@ -385,4 +385,8 @@ archive/ _build/ # Pytest Setup -/module/ \ No newline at end of file +/module/ + +# Testing +seedfarmer.yaml +tmp-metadata diff --git a/CHANGELOG.md b/CHANGELOG.md index 740ebe35..c2107a04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,6 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Changelog](http://keepachangelog.com/). - ## Unreleased ### New @@ -13,6 +12,24 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch ### Fixes +## v3.2.0 (2024-02-26) + +### New +- support list of env files using `--env-file` + +### Changes +- adding `AwsCodeSeederDeployed` and `SeedFarmerDeployed` to all module metadata output for reference (versions used to deploy successfully) +- adding `AWS_CODESEEDER_VERSION` and `SEEDFARMER_VERSION` to all module environment parameters for reference (versions currently in use) +- added `--update-seedkit` support to `apply` + - SeedFarmer will no longer try to update the seedkit on every request + - Users can override this with the `--update-seedkit` flag in case AWS CodeSeeder has updated the SeedKit +- added `--update-project_policy` support to `apply` + - SeedFarmer will apply a changeset to the project policy when this flag is set + +### Fixes +- adding in workaround for manifests whose char length is greater than SSM limit of 8192 k + + ## v3.1.2 (2024-01-24) ### New diff --git a/VERSION b/VERSION index ef538c28..944880fa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.2 +3.2.0 diff --git a/coverage.ini b/coverage.ini deleted file mode 100644 index 703b5c9d..00000000 --- a/coverage.ini +++ /dev/null @@ -1,3 +0,0 @@ -[run] -omit = - test/* \ No newline at end of file diff --git a/docs/source/manifests.md b/docs/source/manifests.md index e67270e5..3fc2d634 100644 --- a/docs/source/manifests.md +++ b/docs/source/manifests.md @@ -364,6 +364,8 @@ In this example, the `glue-db-suffix` parameter will be exposed to the CodeBuild ### Environment Variables `SeedFarmer` supports using [Dotenv](https://github.com/theskumar/python-dotenv) for dynamic replacement. When a file named `.env` is placed at the projecr root (where `seedfarmer.yaml` resides), any value in a manifest with a key of `envVariable` will be matched and replaced with the corresponding environment variable. You can pass in overriding `.env` files by using the `--env-file` on CLI command invocation. +`SeedFarmer` also supports passing multiple `.env`, by using `--env-file` multiple times. For example: `seedfarmer apply --env-file .env.shared --env-file .env.secret`. If the same value is present in multiple `.env` files, subsequent files will override the value from the previous one. In the aforementioned example, values from `.env.secret` will override values from `.env.shared`. + ```yaml name: opensearch path: modules/core/opensearch/ diff --git a/pyproject.toml b/pyproject.toml index 27de2b0e..309f9fe7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] line-length = 120 -target-version = ["py36", "py37", "py38"] +target-version = ["py37", "py38", "py39", "py310", "py311"] exclude = ''' /( \.eggs @@ -27,9 +27,16 @@ use_parentheses = true ensure_newline_before_comments = true line_length = 120 src_paths = ["seedfarmer"] -py_version = 36 +py_version = 37 skip_gitignore = false +[tool.mypy] +python_version = "3.7" +strict = true +ignore_missing_imports = true +disallow_untyped_decorators = false +exclude = "codeseeder.out/|examples/|modules/|test/|seedfarmer.gitmodules/" + [tool.pytest.ini_options] markers = [ "first: marks the first test to run", @@ -77,7 +84,15 @@ markers = [ "commands_bootstrap: marks all `commands_bootstrap` tests", ] log_cli_level = "INFO" -addopts = "-v --cov=. --cov-report=term --cov-report=html --cov-config=coverage.ini --cov-fail-under=80" +addopts = "-v --cov=. --cov-report=term --cov-report=html" pythonpath = [ "." ] + +[tool.pytest.coverage.run] +omit = [ + "test/*" +] + +[tool.coverage.report] +fail_under = 80.0 diff --git a/requirements-dev.in b/requirements-dev.in index 3469895b..79bb9889 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,12 +1,12 @@ -awscli~=1.29.0 -black~=22.3.0 +awscli~=1.31.13 +black~=23.3.0 certifi~=2023.7.22 check-manifest~=0.48 -flake8~=4.0.1 +flake8~=5.0.4 isort~=5.10.1 mypy~=0.961 myst-parser~=0.18.0 -pip-tools~=7.1.0 +pip-tools~=6.14.0 pydot~=1.4.2 pyroma~=4.0 pytest~=7.2.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 5f3d735c..6f14483b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,26 +1,26 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.7 # by the following command: # -# pip-compile requirements-dev.in +# pip-compile --resolver=backtracking requirements-dev.in # alabaster==0.7.13 # via sphinx -astroid==2.15.6 +astroid==2.15.8 # via sphinx-autoapi -attrs==23.1.0 +attrs==23.2.0 # via pytest -awscli==1.29.6 +awscli==1.31.13 # via -r requirements-dev.in -babel==2.12.1 +babel==2.14.0 # via sphinx -black==22.3.0 +black==23.3.0 # via -r requirements-dev.in bleach==6.0.0 # via readme-renderer -boto3==1.28.6 +boto3==1.33.13 # via moto -botocore==1.31.6 +botocore==1.33.13 # via # awscli # boto3 @@ -37,11 +37,11 @@ certifi==2023.7.22 # requests cffi==1.15.1 # via cryptography -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests check-manifest==0.49 # via -r requirements-dev.in -click==8.1.6 +click==8.1.7 # via # black # pip-tools @@ -51,10 +51,8 @@ coverage[toml]==7.2.7 # via # coverage # pytest-cov -cryptography==41.0.6 - # via - # moto - # secretstorage +cryptography==42.0.4 + # via moto docutils==0.16 # via # awscli @@ -65,27 +63,30 @@ docutils==0.16 # sphinx-rtd-theme exceptiongroup==1.2.0 # via pytest -flake8==4.0.1 +flake8==5.0.4 # via -r requirements-dev.in -idna==3.4 +idna==3.6 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==4.2.0 # via + # attrs + # build + # click + # flake8 # keyring + # moto + # pluggy + # pytest # twine iniconfig==2.0.0 # via pytest isort==5.10.1 # via -r requirements-dev.in -jaraco-classes==3.3.0 +jaraco-classes==3.2.3 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage -jinja2==3.1.2 +jinja2==3.1.3 # via # moto # myst-parser @@ -95,7 +96,7 @@ jmespath==1.0.1 # via # boto3 # botocore -keyring==24.2.0 +keyring==23.9.3 # via twine lazy-object-proxy==1.9.0 # via astroid @@ -104,12 +105,12 @@ markdown-it-py==2.2.0 # mdit-py-plugins # myst-parser # rich -markupsafe==2.1.3 +markupsafe==2.1.5 # via # jinja2 # moto # werkzeug -mccabe==0.6.1 +mccabe==0.7.0 # via flake8 mdit-py-plugins==0.3.5 # via myst-parser @@ -129,39 +130,40 @@ mypy-extensions==1.0.0 # mypy myst-parser==0.18.1 # via -r requirements-dev.in -packaging==23.1 +packaging==23.2 # via + # black # build # pyroma # pytest # sphinx -pathspec==0.11.1 +pathspec==0.11.2 # via black -pip-tools==7.1.0 +pip-tools==6.14.0 # via -r requirements-dev.in pkginfo==1.9.6 # via twine -platformdirs==3.9.1 +platformdirs==4.0.0 # via black pluggy==1.2.0 # via pytest -pyasn1==0.5.0 +pyasn1==0.5.1 # via rsa -pycodestyle==2.8.0 +pycodestyle==2.9.1 # via flake8 pycparser==2.21 # via cffi pydot==1.4.2 # via -r requirements-dev.in -pyflakes==2.4.0 +pyflakes==2.5.0 # via flake8 -pygments==2.15.1 +pygments==2.17.2 # via # pyroma # readme-renderer # rich # sphinx -pyparsing==3.1.0 +pyparsing==3.1.1 # via pydot pyproject-hooks==1.0.0 # via build @@ -183,6 +185,8 @@ python-dateutil==2.8.2 # via # botocore # moto +pytz==2024.1 + # via babel pyyaml==6.0.1 # via # awscli @@ -190,7 +194,7 @@ pyyaml==6.0.1 # myst-parser # responses # sphinx-autoapi -readme-renderer==40.0 +readme-renderer==37.3 # via twine requests==2.31.0 # via @@ -203,20 +207,18 @@ requests==2.31.0 # twine requests-toolbelt==1.0.0 # via twine -responses==0.23.1 +responses==0.23.3 # via moto rfc3986==2.0.0 # via twine -rich==13.4.2 +rich==13.7.0 # via twine rsa==4.7.2 # via awscli -s3transfer==0.6.1 +s3transfer==0.8.2 # via # awscli # boto3 -secretstorage==3.3.3 - # via keyring six==1.16.0 # via # bleach @@ -233,11 +235,11 @@ sphinx-autoapi==1.8.4 # via -r requirements-dev.in sphinx-rtd-theme==1.0.0 # via -r requirements-dev.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx @@ -255,11 +257,16 @@ tomli==2.0.1 # pip-tools # pyproject-hooks # pytest -trove-classifiers==2023.7.6 +trove-classifiers==2024.2.22 # via pyroma twine==4.0.2 # via -r requirements-dev.in -types-pyyaml==6.0.12.10 +typed-ast==1.5.5 + # via + # astroid + # black + # mypy +types-pyyaml==6.0.12.12 # via # -r requirements-dev.in # responses @@ -269,9 +276,14 @@ typing-extensions==4.7.1 # via # astroid # black + # importlib-metadata + # markdown-it-py # mypy # myst-parser -unidecode==1.3.6 + # platformdirs + # responses + # rich +unidecode==1.3.8 # via sphinx-autoapi urllib3==1.26.18 # via @@ -282,17 +294,17 @@ urllib3==1.26.18 # twine webencodings==0.5.1 # via bleach -werkzeug==3.0.1 +werkzeug==2.2.3 # via moto wheel==0.38.4 # via # -r requirements-dev.in # pip-tools -wrapt==1.15.0 +wrapt==1.16.0 # via astroid xmltodict==0.13.0 # via moto -zipp==3.16.2 +zipp==3.15.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements.txt b/requirements.txt index b79dbe3e..d2b1c310 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,20 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.7 # by the following command: # # pip-compile # -annotated-types==0.6.0 +annotated-types==0.5.0 # via pydantic arrow==1.2.3 # via jinja2-time -aws-codeseeder==0.11.0 +aws-codeseeder==0.11.1 # via seed-farmer (setup.py) binaryornot==0.4.4 # via cookiecutter -boto3==1.26.125 +boto3==1.33.13 # via aws-codeseeder -botocore==1.29.125 +botocore==1.33.13 # via # boto3 # s3transfer @@ -25,11 +25,11 @@ certifi==2023.7.22 # seed-farmer (setup.py) cfn-flip==1.3.0 # via aws-codeseeder -chardet==5.1.0 +chardet==5.2.0 # via binaryornot -charset-normalizer==3.1.0 +charset-normalizer==3.3.2 # via requests -click==8.1.3 +click==8.1.7 # via # aws-codeseeder # cfn-flip @@ -42,21 +42,25 @@ cookiecutter==2.1.1 # via seed-farmer (setup.py) executor==23.2 # via seed-farmer (setup.py) -fasteners==0.18 +fasteners==0.19 # via executor -gitdb==4.0.10 +gitdb==4.0.11 # via gitpython -gitignore-parser==0.1.3 +gitignore-parser==0.1.11 # via seed-farmer (setup.py) -gitpython==3.1.41 +gitpython==3.1.42 # via seed-farmer (setup.py) humanfriendly==10.0 # via # coloredlogs # executor # property-manager -idna==3.4 +idna==3.6 # via requests +importlib-metadata==6.7.0 + # via + # click + # pydantic jinja2==3.1.3 # via # cookiecutter @@ -67,7 +71,7 @@ jmespath==1.0.1 # via # boto3 # botocore -markupsafe==2.1.2 +markupsafe==2.1.5 # via jinja2 mypy-extensions==1.0.0 # via aws-codeseeder @@ -79,7 +83,7 @@ pydantic==2.5.3 # via seed-farmer (setup.py) pydantic-core==2.14.6 # via pydantic -pygments==2.15.1 +pygments==2.17.2 # via rich pyhumps==3.5.3 # via seed-farmer (setup.py) @@ -89,7 +93,7 @@ python-dateutil==2.8.2 # botocore python-dotenv==0.21.1 # via seed-farmer (setup.py) -python-slugify==8.0.1 +python-slugify==8.0.4 # via cookiecutter pyyaml==6.0.1 # via @@ -103,21 +107,26 @@ requests==2.31.0 # seed-farmer (setup.py) rich==12.4.4 # via seed-farmer (setup.py) -s3transfer==0.6.0 +s3transfer==0.8.2 # via boto3 six==1.16.0 # via # cfn-flip # executor # python-dateutil -smmap==5.0.0 +smmap==5.0.1 # via gitdb text-unidecode==1.3 # via python-slugify typing-extensions==4.6.3 # via + # annotated-types + # arrow + # gitpython + # importlib-metadata # pydantic # pydantic-core + # rich # seed-farmer (setup.py) urllib3==1.26.18 # via @@ -126,3 +135,5 @@ urllib3==1.26.18 # seed-farmer (setup.py) verboselogs==1.7 # via property-manager +zipp==3.15.0 + # via importlib-metadata diff --git a/seedfarmer/__main__.py b/seedfarmer/__main__.py index de38b8cc..792feb62 100644 --- a/seedfarmer/__main__.py +++ b/seedfarmer/__main__.py @@ -14,16 +14,15 @@ import logging -import os -from typing import Optional +from typing import List, Optional import click -from dotenv import load_dotenv import seedfarmer from seedfarmer import DEBUG_LOGGING_FORMAT, commands, config, enable_debug from seedfarmer.cli_groups import bootstrap, init, list, metadata, projectpolicy, remove, store from seedfarmer.output_utils import print_bolded +from seedfarmer.utils import load_dotenv_files _logger: logging.Logger = logging.getLogger(__name__) @@ -65,8 +64,10 @@ def version() -> None: ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -104,24 +105,40 @@ def version() -> None: show_default=True, type=int, ) +@click.option( + "--update-seedkit/--no-update-seedkit", + default=False, + help="Force SeedFarmer to update the SeedKit if found", + show_default=True, + type=bool, +) +@click.option( + "--update-project-policy/--no-update-project-policy", + default=False, + help="Force SeedFarmer to update the deployed Project Policy", + show_default=True, + type=bool, +) def apply( spec: str, profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, dry_run: bool, show_manifest: bool, enable_session_timeout: bool, session_timeout_interval: int, + update_seedkit: bool, + update_project_policy: bool, ) -> None: """Apply manifests to a SeedFarmer managed deployment""" if debug: enable_debug(format=DEBUG_LOGGING_FORMAT) # Load environment variables from .env file if it exists - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files) _logger.info("Apply request with manifest %s", spec) if dry_run: @@ -136,6 +153,8 @@ def apply( show_manifest=show_manifest, enable_session_timeout=enable_session_timeout, session_timeout_interval=session_timeout_interval, + update_seedkit=update_seedkit, + update_project_policy=update_project_policy, ) @@ -177,8 +196,10 @@ def apply( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -208,7 +229,7 @@ def destroy( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, enable_session_timeout: bool, session_timeout_interval: int, @@ -218,7 +239,7 @@ def destroy( enable_debug(format=DEBUG_LOGGING_FORMAT) # Load environment variables from .env file if it exists - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + load_dotenv_files(config.OPS_ROOT, env_files) # MUST use seedfarmer.yaml so we can initialize codeseeder configs project = config.PROJECT diff --git a/seedfarmer/cli_groups/_list_group.py b/seedfarmer/cli_groups/_list_group.py index 05180eac..cb705229 100644 --- a/seedfarmer/cli_groups/_list_group.py +++ b/seedfarmer/cli_groups/_list_group.py @@ -14,12 +14,10 @@ import json import logging -import os import sys -from typing import Optional +from typing import List, Optional import click -from dotenv import load_dotenv import seedfarmer.mgmt.build_info as bi import seedfarmer.mgmt.deploy_utils as du @@ -33,6 +31,7 @@ print_manifest_inventory, ) from seedfarmer.services.session_manager import SessionManager +from seedfarmer.utils import load_dotenv_files _logger: logging.Logger = logging.getLogger(__name__) @@ -110,8 +109,10 @@ def list() -> None: ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -128,7 +129,7 @@ def list_dependencies( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -137,7 +138,8 @@ def list_dependencies( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) @@ -203,8 +205,10 @@ def list_dependencies( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -221,7 +225,7 @@ def list_deployspec( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -230,7 +234,8 @@ def list_deployspec( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -310,8 +315,10 @@ def list_deployspec( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -328,7 +335,7 @@ def list_module_metadata( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], export_local_env: bool, debug: bool, ) -> None: @@ -338,7 +345,8 @@ def list_module_metadata( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -407,8 +415,10 @@ def list_module_metadata( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -423,7 +433,7 @@ def list_all_module_metadata( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -432,7 +442,8 @@ def list_all_module_metadata( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier @@ -500,8 +511,10 @@ def list_all_module_metadata( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -516,7 +529,7 @@ def list_modules( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], debug: bool, ) -> None: if debug: @@ -525,7 +538,9 @@ def list_modules( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) + SessionManager().get_or_create(project_name=project, profile=profile, region_name=region, qualifier=qualifier) dep_manifest = du.generate_deployed_manifest(deployment_name=deployment, skip_deploy_spec=True) @@ -647,8 +662,10 @@ def list_deployments( ) @click.option( "--env-file", - default=".env", + "env_files", + default=[".env"], help="A relative path to the .env file to load environment variables from", + multiple=True, required=False, ) @click.option( @@ -666,7 +683,7 @@ def list_build_env_params( profile: Optional[str], region: Optional[str], qualifier: Optional[str], - env_file: str, + env_files: List[str], export_local_env: str, debug: bool, ) -> None: @@ -678,7 +695,8 @@ def list_build_env_params( if project is None: project = _load_project() - load_dotenv(dotenv_path=os.path.join(config.OPS_ROOT, env_file), verbose=True, override=True) + + load_dotenv_files(config.OPS_ROOT, env_files=env_files) session = SessionManager().get_or_create( project_name=project, profile=profile, region_name=region, qualifier=qualifier diff --git a/seedfarmer/commands/_bootstrap_commands.py b/seedfarmer/commands/_bootstrap_commands.py index 6e8e8d02..eb380910 100644 --- a/seedfarmer/commands/_bootstrap_commands.py +++ b/seedfarmer/commands/_bootstrap_commands.py @@ -88,7 +88,6 @@ def bootstrap_toolchain_account( synthesize: bool = False, as_target: bool = False, ) -> Optional[Dict[Any, Any]]: - if qualifier and not valid_qualifier(qualifier): raise seedfarmer.errors.InvalidConfigurationError("The Qualifier must be alphanumeric and 6 characters or less") @@ -149,7 +148,6 @@ def bootstrap_target_account( policy_arns: Optional[List[str]] = None, synthesize: bool = False, ) -> Optional[Dict[Any, Any]]: - if qualifier and not valid_qualifier(qualifier): raise seedfarmer.errors.InvalidConfigurationError("The Qualifier must be alphanumeric and 6 characters or less") diff --git a/seedfarmer/commands/_deployment_commands.py b/seedfarmer/commands/_deployment_commands.py index 7a7d8997..32a705fc 100644 --- a/seedfarmer/commands/_deployment_commands.py +++ b/seedfarmer/commands/_deployment_commands.py @@ -145,7 +145,6 @@ def _execute_deploy( permissions_boundary_arn: Optional[str] = None, codebuild_image: Optional[str] = None, ) -> ModuleDeploymentResponse: - parameters = load_parameter_values( deployment_name=cast(str, deployment_manifest.name), parameters=module_manifest.parameters, @@ -373,7 +372,9 @@ def _render_permissions_boundary_arn( du.write_deployed_deployment_manifest(deployment_manifest=deployment_manifest) -def prime_target_accounts(deployment_manifest: DeploymentManifest) -> None: +def prime_target_accounts( + deployment_manifest: DeploymentManifest, update_seedkit: bool = False, update_project_policy: bool = False +) -> None: _logger.info("Priming Accounts") with concurrent.futures.ThreadPoolExecutor(max_workers=len(deployment_manifest.target_accounts_regions)) as workers: @@ -385,8 +386,12 @@ def _prime_accounts(args: Dict[str, Any]) -> List[Any]: params = [] for target_account_region in deployment_manifest.target_accounts_regions: - - param_d = {"account_id": target_account_region["account_id"], "region": target_account_region["region"]} + param_d = { + "account_id": target_account_region["account_id"], + "region": target_account_region["region"], + "update_seedkit": update_seedkit, + "update_project_policy": update_project_policy, + } if target_account_region["network"] is not None: network = commands.load_network_values( cast(NetworkMapping, target_account_region["network"]), @@ -394,9 +399,9 @@ def _prime_accounts(args: Dict[str, Any]) -> List[Any]: target_account_region["account_id"], target_account_region["region"], ) - param_d["vpc_id"] = network.vpc_id # type: ignore - param_d["private_subnet_ids"] = network.private_subnet_ids # type: ignore - param_d["security_group_ids"] = network.security_group_ids # type: ignore + param_d["vpc_id"] = network.vpc_id + param_d["private_subnet_ids"] = network.private_subnet_ids + param_d["security_group_ids"] = network.security_group_ids params.append(param_d) @@ -472,6 +477,7 @@ def destroy_deployment( def _exec_destroy(args: Dict[str, Any]) -> Optional[ModuleDeploymentResponse]: return _execute_destroy(**args) + params = [] for _module in _group.modules: _process_module_path(module=_module) if _module.path.startswith("git::") else None @@ -662,6 +668,8 @@ def apply( show_manifest: bool = False, enable_session_timeout: bool = False, session_timeout_interval: int = 900, + update_seedkit: bool = False, + update_project_policy: bool = False, ) -> None: """ apply @@ -695,6 +703,10 @@ def apply( If enabled, boto3 Sessions will be reset on the timeout interval session_timeout_interval: int The interval, in seconds, to reset boto3 Sessions + update_seedkit: bool + Force update run of seedkit, defaults to False + update_project_policy: bool + Force update run of managed project policy, defaults to False Raises ------ @@ -752,7 +764,11 @@ def apply( raise seedfarmer.errors.InvalidPathError("Cannot parse manifest file path") deployment_manifest.validate_and_set_module_defaults() - prime_target_accounts(deployment_manifest=deployment_manifest) + prime_target_accounts( + deployment_manifest=deployment_manifest, + update_seedkit=update_seedkit, + update_project_policy=update_project_policy, + ) module_info_index = du.populate_module_info_index(deployment_manifest=deployment_manifest) destroy_manifest = du.filter_deploy_destroy(deployment_manifest, module_info_index) diff --git a/seedfarmer/commands/_module_commands.py b/seedfarmer/commands/_module_commands.py index a0585d00..fd2fb570 100644 --- a/seedfarmer/commands/_module_commands.py +++ b/seedfarmer/commands/_module_commands.py @@ -19,11 +19,13 @@ import time from typing import Any, Callable, Dict, List, Optional, Tuple, cast +import aws_codeseeder import botocore.exceptions from aws_codeseeder import EnvVar, codeseeder from aws_codeseeder.errors import CodeSeederRuntimeError from boto3 import Session +import seedfarmer import seedfarmer.errors from seedfarmer import config from seedfarmer.commands._runtimes import get_runtimes @@ -75,6 +77,8 @@ def _env_vars( env_vars[_param("PERMISSIONS_BOUNDARY_ARN", use_project_prefix)] = permissions_boundary_arn # Add the partition to env for ease of fetching env_vars["AWS_PARTITION"] = deployment_partition + env_vars["AWS_CODESEEDER_VERSION"] = aws_codeseeder.__version__ + env_vars["SEEDFARMER_VERSION"] = seedfarmer.__version__ return env_vars @@ -119,11 +123,13 @@ def deploy_module( ) ] metadata_env_variable = _param("MODULE_METADATA", use_project_prefix) + sf_version__add = [f"seedfarmer metadata add -k AwsCodeSeederDeployed -v { aws_codeseeder.__version__} || true"] + cs_version_add = [f"seedfarmer metadata add -k SeedFarmerDeployed -v {seedfarmer.__version__} || true"] metadata_put = [ f"if [[ -f {metadata_env_variable} ]]; then export {metadata_env_variable}=$(cat {metadata_env_variable}); fi", ( f"echo ${metadata_env_variable} | seedfarmer store moduledata " - f"-d {deployment_name} -g {group_name} -m {module_manifest.name}" + f"-d {deployment_name} -g {group_name} -m {module_manifest.name} " ), ] @@ -153,7 +159,12 @@ def deploy_module( extra_install_commands=["cd module/"] + _phases.install.commands, extra_pre_build_commands=["cd module/"] + _phases.pre_build.commands, extra_build_commands=["cd module/"] + _phases.build.commands, - extra_post_build_commands=["cd module/"] + _phases.post_build.commands + md5_put + metadata_put, + extra_post_build_commands=["cd module/"] + + _phases.post_build.commands + + md5_put + + sf_version__add + + cs_version_add + + metadata_put, extra_env_vars=env_vars, codebuild_compute_type=module_manifest.deploy_spec.build_type, codebuild_role_name=module_role_name, diff --git a/seedfarmer/commands/_stack_commands.py b/seedfarmer/commands/_stack_commands.py index b6ce0cb4..1b2cf02c 100644 --- a/seedfarmer/commands/_stack_commands.py +++ b/seedfarmer/commands/_stack_commands.py @@ -78,7 +78,11 @@ def _check_stack_status() -> Tuple[bool, Dict[str, str]]: def deploy_managed_policy_stack( - deployment_manifest: DeploymentManifest, account_id: str, region: str, **kwargs: Any + deployment_manifest: DeploymentManifest, + account_id: str, + region: str, + update_project_policy: Optional[bool] = False, + **kwargs: Any, ) -> None: """ deploy_managed_policy_stack @@ -98,7 +102,7 @@ def deploy_managed_policy_stack( project_managed_policy_stack_exists, _ = services.cfn.does_stack_exist( stack_name=info.PROJECT_MANAGED_POLICY_CFN_NAME, session=session ) - if not project_managed_policy_stack_exists: + if not project_managed_policy_stack_exists or update_project_policy: project_managed_policy_template = config.PROJECT_POLICY_PATH _logger.info("Resolved the ProjectPolicyPath %s", project_managed_policy_template) if not os.path.exists(project_managed_policy_template): @@ -435,6 +439,8 @@ def deploy_seedkit( vpc_id: Optional[str] = None, private_subnet_ids: Optional[List[str]] = None, security_group_ids: Optional[List[str]] = None, + update_seedkit: Optional[bool] = False, + **kwargs: Any, ) -> Dict[str, Any]: """ deploy_seedkit @@ -456,20 +462,20 @@ def deploy_seedkit( session = SessionManager().get_or_create().get_deployment_session(account_id=account_id, region_name=region) stack_exists, _, stack_outputs = commands.seedkit_deployed(seedkit_name=config.PROJECT, session=session) deploy_codeartifact = "CodeArtifactRepository" in stack_outputs - if stack_exists: - _logger.debug("Updating SeedKit for Account/Region: %s/%s", account_id, region) + if stack_exists and not update_seedkit: + _logger.debug("SeedKit exists and not updating for Account/Region: %s/%s", account_id, region) else: - _logger.debug("Initializing SeedKit for Account/Region: %s/%s", account_id, region) - commands.deploy_seedkit( - seedkit_name=config.PROJECT, - deploy_codeartifact=deploy_codeartifact, - session=session, - vpc_id=vpc_id, - subnet_ids=private_subnet_ids, - security_group_ids=security_group_ids, - ) - # Go get the outputs and return them - _, _, stack_outputs = commands.seedkit_deployed(seedkit_name=config.PROJECT, session=session) + _logger.debug("Initializing / Updating SeedKit for Account/Region: %s/%s", account_id, region) + commands.deploy_seedkit( + seedkit_name=config.PROJECT, + deploy_codeartifact=deploy_codeartifact, + session=session, + vpc_id=vpc_id, + subnet_ids=private_subnet_ids, + security_group_ids=security_group_ids, + ) + # Go get the outputs and return them + _, _, stack_outputs = commands.seedkit_deployed(seedkit_name=config.PROJECT, session=session) return dict(stack_outputs) @@ -497,7 +503,6 @@ def force_manage_policy_attach( region: str, module_role_name: Optional[str] = None, ) -> None: - session = SessionManager().get_or_create().get_deployment_session(account_id=account_id, region_name=region) if not module_role_name: module_stack_name, module_role_name = get_module_stack_names( diff --git a/seedfarmer/mgmt/deploy_utils.py b/seedfarmer/mgmt/deploy_utils.py index 5db8e88b..871d1d56 100644 --- a/seedfarmer/mgmt/deploy_utils.py +++ b/seedfarmer/mgmt/deploy_utils.py @@ -205,7 +205,6 @@ def validate_module_dependencies( """ def _get_module_list(manifest: DeploymentManifest) -> List[str]: - module_list = [] for group in manifest.groups: for module in group.modules: diff --git a/seedfarmer/mgmt/metadata_support.py b/seedfarmer/mgmt/metadata_support.py index 03ae7436..ca1dbeb2 100644 --- a/seedfarmer/mgmt/metadata_support.py +++ b/seedfarmer/mgmt/metadata_support.py @@ -75,6 +75,16 @@ def _read_metadata_file(mms: ModuleMetadataSupport) -> Dict[str, Any]: return {} +def _read_metadata_env_param(mms: ModuleMetadataSupport) -> Dict[str, Any]: + p = mms.metadata_file_name() + if p in os.environ: + env_data = os.getenv(p) + return cast(Dict[str, Any], json.loads(str(env_data))) + else: + _logger.info("Cannot find existing metadata env param at %s, moving on", p) + return {} + + def _mod_dep_key(mms: ModuleMetadataSupport) -> str: return ( f"{os.getenv(mms.project_param_name())}-" @@ -107,15 +117,26 @@ def _clean_jq(jq: str) -> str: def add_json_output(json_string: str) -> None: mms = ModuleMetadataSupport() - existing_metadata = _read_metadata_file(mms=mms) json_new = json.loads(json_string) - _write_metadata_file(mms=mms, data={**json_new, **existing_metadata}) + file_dict = _read_metadata_file(mms=mms) + json_new = {**json_new, **file_dict} if file_dict else json_new + _logger.debug(f"Current Dict {json.dumps(json_new, indent=4)}") + env_dict = _read_metadata_env_param(mms=mms) + json_new = {**json_new, **env_dict} if env_dict else json_new + _logger.debug(f"Current Dict {json.dumps(json_new, indent=4)}") + _write_metadata_file(mms=mms, data=json_new) def add_kv_output(key: str, value: str) -> None: mms = ModuleMetadataSupport() - data = _read_metadata_file(mms=mms) + data = {} data[key] = value + file_dict = _read_metadata_file(mms=mms) + data = {**data, **file_dict} if file_dict else data + _logger.debug(f"Current Dict {json.dumps(data, indent=4)}") + env_dict = _read_metadata_env_param(mms=mms) + data = {**data, **env_dict} if env_dict else data + _logger.debug(f"Current Dict {json.dumps(data, indent=4)}") _write_metadata_file(mms=mms, data=data) diff --git a/seedfarmer/mgmt/module_info.py b/seedfarmer/mgmt/module_info.py index c1d69929..203af43e 100644 --- a/seedfarmer/mgmt/module_info.py +++ b/seedfarmer/mgmt/module_info.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import os +import sys from enum import Enum from typing import Any, Dict, List, Optional, Tuple @@ -23,7 +25,7 @@ from seedfarmer import config from seedfarmer.services import _secrets_manager as secrets from seedfarmer.services import _ssm as ssm -from seedfarmer.utils import generate_hash, generate_session_hash +from seedfarmer.utils import generate_hash, generate_session_hash, remove_nulls _logger: logging.Logger = logging.getLogger(__name__) @@ -538,7 +540,15 @@ def write_module_manifest( session: Session, optional The boto3.Session to use to for SSM Parameter queries, default None """ - ssm.put_parameter(name=_manifest_key(deployment, group, module), obj=data, session=session) + + # Temp fix until a larger persistence store is vetted + process_data = data + current_size = sys.getsizeof(json.dumps(data)) + if current_size > 8191: + _logger.info("The manifest for %s-%s is %s, too large for SSM, reducing", group, module, current_size) + process_data = remove_nulls(data) + _logger.info("The size is now %s", sys.getsizeof(json.dumps(process_data))) + ssm.put_parameter(name=_manifest_key(deployment, group, module), obj=process_data, session=session) def write_deployspec( diff --git a/seedfarmer/models/deploy_responses.py b/seedfarmer/models/deploy_responses.py index 132d48d2..eb9f7e63 100644 --- a/seedfarmer/models/deploy_responses.py +++ b/seedfarmer/models/deploy_responses.py @@ -27,7 +27,6 @@ class StatusType(Enum): class CodeSeederMetadata(CamelModel): - _build_url: str = PrivateAttr() aws_account_id: Optional[str] = None @@ -51,7 +50,6 @@ def build_url(self) -> str: class ModuleDeploymentResponse(CamelModel): - deployment: str group: Optional[str] = None module: str diff --git a/seedfarmer/services/_service_utils.py b/seedfarmer/services/_service_utils.py index 111f27fe..4fc4fd72 100644 --- a/seedfarmer/services/_service_utils.py +++ b/seedfarmer/services/_service_utils.py @@ -104,7 +104,6 @@ def get_region(session: Optional[Session] = None, profile: Optional[str] = None) def _call_sts(session: Optional[Session] = None, profile: Optional[str] = None) -> Dict[str, Any]: - try: if not session: return cast(Dict[str, Any], boto3_client(service_name="sts", profile=profile).get_caller_identity()) diff --git a/seedfarmer/utils.py b/seedfarmer/utils.py index 54e9417a..5dffc571 100644 --- a/seedfarmer/utils.py +++ b/seedfarmer/utils.py @@ -14,11 +14,13 @@ import hashlib import logging -from typing import Optional +import os +from typing import Any, Dict, List, Optional import humps import yaml from boto3 import Session +from dotenv import dotenv_values, load_dotenv from seedfarmer.services._service_utils import get_region, get_sts_identity_info @@ -158,3 +160,35 @@ def get_deployment_role_arn( def valid_qualifier(qualifer: str) -> bool: return True if ((len(qualifer) <= 6) and qualifer.isalnum()) else False + + +def load_dotenv_files(root_path: str, env_files: List[str]) -> None: + """ + Load the environment variables from the .env files + + Parameters + ---------- + root_path : str + The path to the root of the project + env_files : List[str] + The list of the .env files to load + """ + loaded_values = {} + + for env_file in env_files: + _logger.info("Loading environment variables from %s", env_file) + dotenv_path = os.path.join(root_path, env_file) + + load_dotenv(dotenv_path=dotenv_path, verbose=True, override=True) + loaded_values.update(dotenv_values(dotenv_path, verbose=True)) + + _logger.debug("Loaded environment variables: %s", loaded_values) + + +def remove_nulls(payload: Dict[str, Any]) -> Dict[str, Any]: + if isinstance(payload, dict): + return {k: remove_nulls(v) for k, v in payload.items() if v is not None} + elif isinstance(payload, list): + return [remove_nulls(v) for v in payload] + else: + return payload diff --git a/setup.cfg b/setup.cfg index 0e163151..874312cf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,11 +18,3 @@ exclude = codeseeder.out, bundle, seedfarmer.gitmodules - -[mypy] -python_version = 3.7 -strict = True -ignore_missing_imports = True -allow_untyped_decorators = True -exclude = - codeseeder.out/|examples/|modules/|test/|seedfarmer.gitmodules/ \ No newline at end of file diff --git a/test/unit-test/test_cli_arg.py b/test/unit-test/test_cli_arg.py index 91c92db8..5006205b 100644 --- a/test/unit-test/test_cli_arg.py +++ b/test/unit-test/test_cli_arg.py @@ -48,6 +48,32 @@ def aws_credentials(): os.environ["MOTO_ACCOUNT_ID"] = "123456789012" +@pytest.fixture(scope="function") +def env_file(): + path = os.path.join(_OPS_ROOT, ".env") + + with open(path, "w") as f: + f.write("PRIMARY_ACCOUNT=123456789012\n") + f.write("VPCID=vpc-123456\n") + + yield path + + os.remove(path) + + +@pytest.fixture(scope="function") +def env_file2(): + path = os.path.join(_OPS_ROOT, ".env.test2") + + with open(path, "w") as f: + f.write("PRIMARY_ACCOUNT=000000000000\n") + f.write("SECONDARY_ACCOUNT=123456789012\n") + + yield path + + os.remove(path) + + @pytest.fixture(scope="function") def sts_client(aws_credentials): with mock_sts(): @@ -161,6 +187,55 @@ def test_apply_deployment(mocker): command_output = _test_command(sub_command=apply, options=[deployment_manifest, "--debug"], exit_code=0) +@pytest.mark.first +@pytest.mark.apply_working_module +def test_apply_deployment__env_variables_no_env_file(mocker, env_file): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + _test_command(sub_command=apply, options=[deployment_manifest, "--debug"], exit_code=0) + + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} + + +@pytest.mark.first +@pytest.mark.apply_working_module +def test_apply_deployment__env_variables_single_env_file(mocker, env_file): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_file], exit_code=0) + + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} + + +@pytest.mark.first +@pytest.mark.apply_working_module +@pytest.mark.parametrize("reverse_order", [False, True]) +def test_apply_deployment__env_variables_multiple_env_files(mocker, reverse_order, env_file, env_file2): + # Deploys a functioning module + mocker.patch("seedfarmer.__main__.commands.apply", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + deployment_manifest = f"{_TEST_ROOT}/manifests/module-test/deployment.yaml" + + env_files = [env_file, env_file2] + if reverse_order: + env_files = env_files[::-1] + + _test_command(sub_command=apply, options=[deployment_manifest, "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) + + assert os.environ == { + "PRIMARY_ACCOUNT": "123456789012" if reverse_order else "000000000000", + "SECONDARY_ACCOUNT": "123456789012", + "VPCID": "vpc-123456", + } + + @pytest.mark.destroy def test_destroy_deployment_dry_run(mocker): # Destroy a functioning module @@ -175,6 +250,48 @@ def test_destroy_deployment(mocker): command_output = _test_command(sub_command=destroy, options=["myapp", "--debug"], exit_code=0) +@pytest.mark.destroy +def test_destroy__deployment_env_variables_no_env_file(mocker, env_file): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + _test_command(sub_command=destroy, options=["myapp", "--debug"], exit_code=0) + + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} + + +@pytest.mark.destroy +def test_destroy__deployment_env_variables_single_env_file(mocker, env_file): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_file], exit_code=0) + + assert os.environ == {"PRIMARY_ACCOUNT": "123456789012", "VPCID": "vpc-123456"} + + +@pytest.mark.destroy +@pytest.mark.parametrize("reverse_order", [False, True]) +def test_destroy__deployment_env_variables_multiple_env_files(mocker, reverse_order, env_file, env_file2): + # Destroy a functioning module + mocker.patch("seedfarmer.__main__.commands.destroy", return_value=None) + mocker.patch.dict(os.environ, {}, clear=True) + + env_files = [env_file, env_file2] + if reverse_order: + env_files = env_files[::-1] + + _test_command(sub_command=destroy, options=["myapp", "--debug", "--env-file", env_files[0], "--env-file", env_files[1]], exit_code=0) + + assert os.environ == { + "PRIMARY_ACCOUNT": "123456789012" if reverse_order else "000000000000", + "SECONDARY_ACCOUNT": "123456789012", + "VPCID": "vpc-123456", + } + + @pytest.mark.bootstrap def test_bootstrap_toolchain_only(mocker): # Bootstrap an Account As Target diff --git a/test/unit-test/test_mgmt_module_info.py b/test/unit-test/test_mgmt_module_info.py index 0d1d3b3c..363e5491 100644 --- a/test/unit-test/test_mgmt_module_info.py +++ b/test/unit-test/test_mgmt_module_info.py @@ -246,7 +246,8 @@ def test_write_module_manifest(aws_credentials, session, mocker): import seedfarmer.mgmt.module_info as mi mocker.patch("seedfarmer.mgmt.module_info.ssm.put_parameter", return_value=True) - mi.write_module_manifest(deployment="myapp", group="test", module="mymodule", data={"Hey", "Yo"}, session=session) + payload = {"Hey", "Yo"} + mi.write_module_manifest(deployment="myapp", group="test", module="mymodule", data=dict.fromkeys(payload, 0), session=session) @pytest.mark.mgmt