From 9099ec4c2263ff4148f3432ecb77d51339aa5982 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin Date: Sun, 29 Sep 2024 19:20:49 +0100 Subject: [PATCH 1/5] add param to disable env replacement in module params --- seedfarmer/models/manifests/_module_manifest.py | 1 + seedfarmer/utils.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/seedfarmer/models/manifests/_module_manifest.py b/seedfarmer/models/manifests/_module_manifest.py index 008ad9e..fa20f1a 100644 --- a/seedfarmer/models/manifests/_module_manifest.py +++ b/seedfarmer/models/manifests/_module_manifest.py @@ -29,6 +29,7 @@ class ModuleParameter(ValueFromRef): name: str value: Optional[Any] = None version: Optional[Any] = None + disable_env_var_resolution: Optional[bool] = None resolved_value: Optional[Any] = None def __init__(self, **data: Any) -> None: diff --git a/seedfarmer/utils.py b/seedfarmer/utils.py index ef6eb3f..d5242cc 100644 --- a/seedfarmer/utils.py +++ b/seedfarmer/utils.py @@ -237,13 +237,14 @@ def recurse_list(working_list: List[Any]) -> List[Any]: return working_list def recurse_dict(working_element: Dict[str, Any]) -> Dict[str, Any]: - for key, value in working_element.items(): - if isinstance(value, str): - working_element[key] = replace_str(value) - elif isinstance(value, list): - recurse_list(value) - elif isinstance(value, dict) and key not in ["deploy_spec"]: - recurse_dict(value) + if not working_element.get("disable_env_var_resolution"): + for key, value in working_element.items(): + if isinstance(value, str): + working_element[key] = replace_str(value) + elif isinstance(value, list): + recurse_list(value) + elif isinstance(value, dict) and key not in ["deploy_spec"]: + recurse_dict(value) return working_element payload = recurse_dict(payload) From 3b93366c08636eda3f4ade7fe270ab238bdf5577 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin Date: Sun, 29 Sep 2024 19:21:08 +0100 Subject: [PATCH 2/5] testcases --- test/unit-test/test_models.py | 75 +++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/test/unit-test/test_models.py b/test/unit-test/test_models.py index 7542cc7..9d43b68 100644 --- a/test/unit-test/test_models.py +++ b/test/unit-test/test_models.py @@ -637,3 +637,78 @@ def test_module_manifest_with_env_var_resolution_in_path_error(): ): with pytest.raises(InvalidManifestError, match="The environment variable is not available: 'GIT_URL'"): ModuleManifest(**module_yaml) + + +@pytest.mark.models +@pytest.mark.models_module_manifest +def test_module_manifest_with_env_var_resolution_in_script(): + module_yaml = yaml.safe_load( + """ +name: test-module-1 +path: modules/test-module +targetAccount: primary +targetRegion: us-west-2 +parameters: + - name: param1 + value: + - | + echo "${VAR}" +""" + ) + + with mock.patch.dict( + os.environ, + { + "VAR": "test", + }, + clear=True, + ): + ModuleManifest(**module_yaml) + + +@pytest.mark.models +@pytest.mark.models_module_manifest +def test_module_manifest_with_env_var_resolution_in_script_disabled(): + module_yaml = yaml.safe_load( + """ +name: test-module-1 +path: modules/test-module +targetAccount: primary +targetRegion: us-west-2 +parameters: + - name: param1 + disable_env_var_resolution: True + value: + - | + export VAR=test + echo "${VAR}" + - name: param2_nested + disable_env_var_resolution: True + value: + nested_value: + - | + export VAR=test + echo "${VAR}" +""" + ) + ModuleManifest(**module_yaml) + + +@pytest.mark.models +@pytest.mark.models_module_manifest +def test_module_manifest_with_env_var_resolution_in_script_error(): + module_yaml = yaml.safe_load( + """ +name: test-module-1 +path: modules/test-module +targetAccount: primary +targetRegion: us-west-2 +parameters: + - name: param1 + value: + - | + echo "${VAR}" +""" + ) + with pytest.raises(InvalidManifestError, match="The environment variable is not available: 'VAR'"): + ModuleManifest(**module_yaml) From d67785a449d87af818b8b5aad4572c8ee2c7d672 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin Date: Sun, 29 Sep 2024 19:22:15 +0100 Subject: [PATCH 3/5] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d731fa3..5e588bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a Ch ### Changes +- Added ability to disable env replacement in module parameters + ### Fixes ## v5.0.0 (2024-08-16) From 3885ffb58319c1c56ab4a1bbc3ebad214aa280a4 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin Date: Sun, 29 Sep 2024 19:39:11 +0100 Subject: [PATCH 4/5] use snake case --- seedfarmer/models/manifests/_module_manifest.py | 2 +- seedfarmer/utils.py | 2 +- test/unit-test/test_models.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/seedfarmer/models/manifests/_module_manifest.py b/seedfarmer/models/manifests/_module_manifest.py index fa20f1a..af920ec 100644 --- a/seedfarmer/models/manifests/_module_manifest.py +++ b/seedfarmer/models/manifests/_module_manifest.py @@ -29,7 +29,7 @@ class ModuleParameter(ValueFromRef): name: str value: Optional[Any] = None version: Optional[Any] = None - disable_env_var_resolution: Optional[bool] = None + disableEnvVarResolution: Optional[bool] = None resolved_value: Optional[Any] = None def __init__(self, **data: Any) -> None: diff --git a/seedfarmer/utils.py b/seedfarmer/utils.py index d5242cc..818afd6 100644 --- a/seedfarmer/utils.py +++ b/seedfarmer/utils.py @@ -237,7 +237,7 @@ def recurse_list(working_list: List[Any]) -> List[Any]: return working_list def recurse_dict(working_element: Dict[str, Any]) -> Dict[str, Any]: - if not working_element.get("disable_env_var_resolution"): + if not working_element.get("disableEnvVarResolution"): for key, value in working_element.items(): if isinstance(value, str): working_element[key] = replace_str(value) diff --git a/test/unit-test/test_models.py b/test/unit-test/test_models.py index 9d43b68..19b4f2d 100644 --- a/test/unit-test/test_models.py +++ b/test/unit-test/test_models.py @@ -677,13 +677,13 @@ def test_module_manifest_with_env_var_resolution_in_script_disabled(): targetRegion: us-west-2 parameters: - name: param1 - disable_env_var_resolution: True + disableEnvVarResolution: True value: - | export VAR=test echo "${VAR}" - name: param2_nested - disable_env_var_resolution: True + disableEnvVarResolution: True value: nested_value: - | From df4506da9ba0837dd0418c9341484e9858062db3 Mon Sep 17 00:00:00 2001 From: Anton Kukushkin Date: Sun, 29 Sep 2024 19:39:33 +0100 Subject: [PATCH 5/5] update docs --- docs/source/manifests.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/manifests.md b/docs/source/manifests.md index c6ba458..5bbc44d 100644 --- a/docs/source/manifests.md +++ b/docs/source/manifests.md @@ -330,7 +330,7 @@ When using this feature, any change to these file(s) (modifying, add to manifest ## Universal Environment Variable Replacement in Manifests As of the release of `seed-farmer==3.5.0`, we have added support for dynamic replacement of values with environment variables in manifests. This does not replace any pre-existing functionality. This also is limited to only manifests (`deployment_manifest` and `module_manifest`). Things like the `deployspec` and the `modulestack` are NOT included in this functionality. We strongly recommend using hard-coded values in manifests or leveraging the facilities already in place, but we have added this feature based on feedback from experienced users. -Any string within your manifests that has a designated pattern will automatically be resolved. If you have an environment variable named `SOMEKEY` that is defined, you can reference it in your manifests via wrapping it in `${}` --> for example `${SOMEKEY}`. +Any string within your manifests that has a designated pattern will automatically be resolved. If you have an environment variable named `SOMEKEY` that is defined, you can reference it in your manifests via wrapping it in `${}` --> for example `${SOMEKEY}`. Additionally, it is possible to disable environment variable replacement in module input parameters using `disableEnvVarResolution: True` for cases such as when input parameter is a script. The following is a valid manifest: @@ -356,6 +356,12 @@ parameters: - name: vpc-id valueFrom: secretsManager: ${SOMEKEY} + - name: param-no-env-resolution + disableEnvVarResolution: True + value: + - | + export VAR=test + echo "${VAR}" ``` This can be applied to all values in the manifest. We do not recommend using this in the `name` field of manifests as any value that is referenced by downstream manifests MUST align. For example, in the following: