Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updating as per dependabots #540

Merged
merged 8 commits into from
Apr 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
python-version: ["3.8", "3.9", "3.10", "3.11"]
defaults:
run:
working-directory: .
Expand Down
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch
- adding info for destroy and list deployments when no deployments found
- refactored git support logic to separate python file
- added verbose messaging related to git issues
- updated `black~=24.3.0` in requirements-dev as per dependabot
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be also note here that we're dropping Python 3.7 support?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmmmm...wondering. The only reason I removed it was due to black...but, of course I removed the support in setup..
Adding comment to changelog

- updated `werkzeug~=2.3.8` in requirements-dev as per dependabot
- removed explicit support for python 3.7
- this was only due to support for black in requirements-dev, but was also removed from setup.py


### Fixes
- Add schema validation step checking that either `value` or `value_from` is present for each parameter
Expand Down
3 changes: 2 additions & 1 deletion requirements-dev.in
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
awscli~=1.31.13
black~=23.3.0
black~=24.3.0
certifi~=2023.7.22
check-manifest~=0.48
flake8~=5.0.4
Expand All @@ -22,4 +22,5 @@ sphinx-rtd-theme~=1.0.0
sphinx~=4.3.0
moto[s3,sts,iam,codebuild,secretsmanager,ssm]~=4.0.9
requests~=2.31.0
werkzeug~=2.3.8
urllib3~=1.26.18
39 changes: 9 additions & 30 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#
# This file is autogenerated by pip-compile with Python 3.7
# This file is autogenerated by pip-compile with Python 3.9
# by the following command:
#
# pip-compile --resolver=backtracking requirements-dev.in
# pip-compile requirements-dev.in
#
alabaster==0.7.13
# via sphinx
Expand All @@ -14,7 +14,7 @@ awscli==1.31.13
# via -r requirements-dev.in
babel==2.14.0
# via sphinx
black==23.3.0
black==24.3.0
# via -r requirements-dev.in
bleach==6.0.0
# via readme-renderer
Expand Down Expand Up @@ -48,9 +48,7 @@ click==8.1.7
colorama==0.4.4
# via awscli
coverage[toml]==7.2.7
# via
# coverage
# pytest-cov
# via pytest-cov
cryptography==42.0.4
# via moto
docutils==0.16
Expand All @@ -71,14 +69,7 @@ imagesize==1.4.1
# via sphinx
importlib-metadata==4.2.0
# via
# attrs
# build
# click
# flake8
# keyring
# moto
# pluggy
# pytest
# twine
iniconfig==2.0.0
# via pytest
Expand Down Expand Up @@ -119,9 +110,7 @@ mdurl==0.1.2
more-itertools==9.1.0
# via jaraco-classes
moto[codebuild,iam,s3,secretsmanager,ssm,sts]==4.0.13
# via
# -r requirements-dev.in
# moto
# via -r requirements-dev.in
mypy==0.991
# via -r requirements-dev.in
mypy-extensions==1.0.0
Expand Down Expand Up @@ -185,8 +174,6 @@ python-dateutil==2.8.2
# via
# botocore
# moto
pytz==2024.1
# via babel
pyyaml==6.0.1
# via
# awscli
Expand Down Expand Up @@ -261,11 +248,6 @@ trove-classifiers==2024.2.22
# via pyroma
twine==4.0.2
# via -r requirements-dev.in
typed-ast==1.5.5
# via
# astroid
# black
# mypy
types-pyyaml==6.0.12.12
# via
# -r requirements-dev.in
Expand All @@ -276,13 +258,8 @@ typing-extensions==4.7.1
# via
# astroid
# black
# importlib-metadata
# markdown-it-py
# mypy
# myst-parser
# platformdirs
# responses
# rich
unidecode==1.3.8
# via sphinx-autoapi
urllib3==1.26.18
Expand All @@ -294,8 +271,10 @@ urllib3==1.26.18
# twine
webencodings==0.5.1
# via bleach
werkzeug==2.2.3
# via moto
werkzeug==2.3.8
# via
# -r requirements-dev.in
# moto
wheel==0.38.4
# via
# -r requirements-dev.in
Expand Down
8 changes: 5 additions & 3 deletions seedfarmer/checksum.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,11 @@ def scandir(dirname: str) -> List[str]:
subfolders = [f.path for f in os.scandir(dirname) if f.is_dir()]
for dirname in list(subfolders):
# ignore all hidden directories and any dir already in .gitignore
subfolders.extend(scandir(dirname)) if not os.path.split(dirname)[1].startswith(".") and not _evaluate_file(
dirname, ignore_maps
) else None
(
subfolders.extend(scandir(dirname))
if not os.path.split(dirname)[1].startswith(".") and not _evaluate_file(dirname, ignore_maps)
else None
)
return subfolders

_ = scandir(os.path.join(project_path, module_path))
Expand Down
24 changes: 16 additions & 8 deletions seedfarmer/cli_groups/_list_group.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,14 +152,22 @@ def list_dependencies(

if dep_manifest:
module_depends_on_dict, module_dependencies_dict = du.generate_dependency_maps(manifest=dep_manifest)
print_dependency_list(
header_message=f"Modules that {module} in {group} of {deployment} DEPENDS ON : ",
modules=module_depends_on_dict[f"{group}-{module}"],
) if module_depends_on_dict.get(f"{group}-{module}") else None
print_dependency_list(
header_message=f"Modules that ARE DEPENDENT ON {module} in {group} of {deployment} : ",
modules=module_dependencies_dict[f"{group}-{module}"],
) if module_dependencies_dict.get(f"{group}-{module}") else None
(
print_dependency_list(
header_message=f"Modules that {module} in {group} of {deployment} DEPENDS ON : ",
modules=module_depends_on_dict[f"{group}-{module}"],
)
if module_depends_on_dict.get(f"{group}-{module}")
else None
)
(
print_dependency_list(
header_message=f"Modules that ARE DEPENDENT ON {module} in {group} of {deployment} : ",
modules=module_dependencies_dict[f"{group}-{module}"],
)
if module_dependencies_dict.get(f"{group}-{module}")
else None
)


@list.command(name="deployspec", help="List the stored deployspec of a module")
Expand Down
54 changes: 35 additions & 19 deletions seedfarmer/commands/_deployment_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,13 +124,17 @@ def _execute_deploy(
f"Invalid value for ModuleManifest.deploy_spec in group {group_name} and module : {module_manifest.name}"
)

du.prepare_ssm_for_deploy(
deployment_name=deployment_manifest.name,
group_name=group_name,
module_manifest=module_manifest,
account_id=target_account_id,
region=target_region,
) if deployment_manifest.name else None
(
du.prepare_ssm_for_deploy(
deployment_name=deployment_manifest.name,
group_name=group_name,
module_manifest=module_manifest,
account_id=target_account_id,
region=target_region,
)
if deployment_manifest.name
else None
)

return commands.deploy_module(
deployment_name=cast(str, deployment_manifest.name),
Expand Down Expand Up @@ -450,9 +454,13 @@ def _exec_destroy(args: Dict[str, Any]) -> Optional[ModuleDeploymentResponse]:
for _module in _group.modules:
_process_module_path(module=_module) if _module.path.startswith("git::") else None

_process_data_files(
data_files=_module.data_files, module_name=_module.name, group_name=_group.name
) if _module.data_files is not None else None
(
_process_data_files(
data_files=_module.data_files, module_name=_module.name, group_name=_group.name
)
if _module.data_files is not None
else None
)

if _module and _module.deploy_spec:
params = [
Expand All @@ -474,9 +482,11 @@ def _exec_destroy(args: Dict[str, Any]) -> Optional[ModuleDeploymentResponse]:
]
destroy_response = list(workers.map(_exec_destroy, params))
_logger.debug(destroy_response)
print_modules_build_info("Build Info Debug Data", destroy_response) if _logger.isEnabledFor(
logging.DEBUG
) else None
(
print_modules_build_info("Build Info Debug Data", destroy_response)
if _logger.isEnabledFor(logging.DEBUG)
else None
)
for dep_resp_object in destroy_response:
if dep_resp_object and dep_resp_object.status in ["ERROR", "error", "Error"]:
_logger.error("At least one module failed to destroy...exiting deployment")
Expand Down Expand Up @@ -555,9 +565,11 @@ def deploy_deployment(

_process_module_path(module=module) if module.path.startswith("git::") else None

_process_data_files(
data_files=module.data_files, module_name=module.name, group_name=group.name
) if module.data_files is not None else None
(
_process_data_files(data_files=module.data_files, module_name=module.name, group_name=group.name)
if module.data_files is not None
else None
)

deployspec_path = get_deployspec_path(str(module.get_local_path()))
with open(deployspec_path) as module_spec_file:
Expand Down Expand Up @@ -615,9 +627,13 @@ def deploy_deployment(
name=group.name, path=group.path, concurrency=group.concurrency, modules=modules_to_deploy
)
)
_print_modules(
f"Modules deployed that are up to date (will not be changed): {deployment_name} ", unchanged_modules
) if unchanged_modules else None
(
_print_modules(
f"Modules deployed that are up to date (will not be changed): {deployment_name} ", unchanged_modules
)
if unchanged_modules
else None
)
_deploy_validated_deployment(
deployment_manifest=deployment_manifest,
deployment_manifest_wip=deployment_manifest_wip,
Expand Down
6 changes: 3 additions & 3 deletions seedfarmer/commands/_module_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ def _env_vars(
) -> Dict[str, str]:
env_vars = (
{
f"{_param('PARAMETER', use_project_prefix)}_{p.upper_snake_case}": p.value
if isinstance(p.value, str) or isinstance(p.value, EnvVar)
else json.dumps(p.value)
f"{_param('PARAMETER', use_project_prefix)}_{p.upper_snake_case}": (
p.value if isinstance(p.value, str) or isinstance(p.value, EnvVar) else json.dumps(p.value)
)
for p in parameters
}
if parameters
Expand Down
58 changes: 35 additions & 23 deletions seedfarmer/mgmt/deploy_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,29 +296,41 @@ def prepare_ssm_for_deploy(
data=module_manifest_wip.model_dump(),
session=session,
)
mi.write_deployspec(
deployment=deployment_name,
group=group_name,
module=module_manifest.name,
data=module_manifest.deploy_spec.model_dump(),
session=session,
) if module_manifest.deploy_spec else None
mi.write_module_md5(
deployment=deployment_name,
group=group_name,
module=module_manifest.name,
hash=module_manifest.deployspec_md5,
type=mi.ModuleConst.DEPLOYSPEC,
session=session,
) if module_manifest.deployspec_md5 else None
mi.write_module_md5(
deployment=deployment_name,
group=group_name,
module=module_manifest.name,
hash=module_manifest.manifest_md5,
type=mi.ModuleConst.MANIFEST,
session=session,
) if module_manifest.manifest_md5 else None
(
mi.write_deployspec(
deployment=deployment_name,
group=group_name,
module=module_manifest.name,
data=module_manifest.deploy_spec.model_dump(),
session=session,
)
if module_manifest.deploy_spec
else None
)
(
mi.write_module_md5(
deployment=deployment_name,
group=group_name,
module=module_manifest.name,
hash=module_manifest.deployspec_md5,
type=mi.ModuleConst.DEPLOYSPEC,
session=session,
)
if module_manifest.deployspec_md5
else None
)
(
mi.write_module_md5(
deployment=deployment_name,
group=group_name,
module=module_manifest.name,
hash=module_manifest.manifest_md5,
type=mi.ModuleConst.MANIFEST,
session=session,
)
if module_manifest.manifest_md5
else None
)
mi.remove_module_md5(
deployment=deployment_name,
group=group_name,
Expand Down
16 changes: 10 additions & 6 deletions seedfarmer/output_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,11 @@ def print_modules_build_info(
console.print(
f" [cyan]{r_obj.deployment}-{r_obj.group}-{r_obj.module} status: {r_obj.status}", crop=False # type: ignore
)
console.print(
f" {r_obj.codeseeder_metadata.build_url}", crop=False # type: ignore
) if r_obj.codeseeder_metadata and r_obj.codeseeder_metadata.build_url else None # type: ignore
(
console.print(f" {r_obj.codeseeder_metadata.build_url}", crop=False) # type: ignore
if r_obj.codeseeder_metadata and r_obj.codeseeder_metadata.build_url # type: ignore
else None
)


def print_errored_modules_build_info(
Expand All @@ -182,9 +184,11 @@ def print_errored_modules_build_info(
for r_obj in modules_data:
if r_obj and r_obj.status in ["ERROR", "error", "Error"]:
console.print(f" [cyan]{r_obj.deployment}-{r_obj.group}-{r_obj.module}", crop=False)
console.print(
f" {r_obj.codeseeder_metadata.build_url}", crop=False
) if r_obj.codeseeder_metadata and r_obj.codeseeder_metadata.build_url else None
(
console.print(f" {r_obj.codeseeder_metadata.build_url}", crop=False)
if r_obj.codeseeder_metadata and r_obj.codeseeder_metadata.build_url
else None
)


def print_dependency_error_list(header_message: str, errored_list: List[Dict[str, List[str]]]) -> None:
Expand Down
Loading