From 5c5f01ff1d7802ddff2343570877092e268d3c3b Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Fri, 28 Feb 2025 07:06:54 -0700 Subject: [PATCH 1/3] Ci testing on aws 2 (#3408) The CI testing Jenkinsfile4AWS has a syntax error, and also need to change the work directory. Refs #3341 --- ci/Jenkinsfile4AWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index c23ace3a96..5efc18bf89 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -6,7 +6,7 @@ def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: /lustre/jenkins/global-workflow/CI] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' From 7fe03e670ad55d9e66b1bde4d954c220dbfd1e27 Mon Sep 17 00:00:00 2001 From: Anna Shlyaeva Date: Fri, 28 Feb 2025 07:08:10 -0700 Subject: [PATCH 2/3] Copy marine ensemble output observation diags and spread (#3407) Saves to COMROOT marine LETKF obs diagnostics files (for ensemble spread diagnostics in the observation space) and sets up additional keys to pass to the gdasapp jinja template for copying model space files (for ensemble mean and spread diagnostics in the model space). Resolves #3363 --- ush/python/pygfs/task/marine_letkf.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 98c4f29085..02f4e2a7a4 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -184,8 +184,26 @@ def finalize(self): letkfsaveconf = AttrDict() keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN', 'GDUMP_ENS', - 'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL'] + 'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL', + 'COMOUT_OCEAN_LETKF', 'COMOUT_ICE_LETKF', 'WINDOW_MIDDLE', + 'MARINE_OBS_LIST_YAML'] for key in keys: letkfsaveconf[key] = self.task_config[key] + + # get the list of obs output files + obs_list = parse_j2yaml(letkfsaveconf.MARINE_OBS_LIST_YAML, self.task_config) + obs_files = [] + for ob in obs_list['observers']: + obs_files.append(ob['obs space']['obsdataout']['engine']['obsfile']) + obs_files_to_copy = [] + # copy obs from diags to COMOUT + for obs_src in obs_files: + obs_dst = os.path.join(letkfsaveconf.COMOUT_OCEAN_LETKF, 'diags', + os.path.basename(obs_src)) + if os.path.exists(obs_src): + obs_files_to_copy.append([obs_src, obs_dst]) + # stage the desired diag files + FileHandler({'mkdir': [os.path.join(letkfsaveconf.COMOUT_OCEAN_LETKF, 'diags')]}).sync() + FileHandler({'copy': obs_files_to_copy}).sync() letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf) FileHandler(letkf_save_list).sync() From bbd5cca163037a36b833582ef77bc3f6bbe41452 Mon Sep 17 00:00:00 2001 From: mingshichen-noaa <48537176+mingshichen-noaa@users.noreply.github.com> Date: Fri, 28 Feb 2025 09:10:13 -0500 Subject: [PATCH 3/3] Update enkf archive jobs to use COMIN/COMOUT (#3393) # Description NCO has requested that each COM variable specify whether it is an input or an output. This completes that process for the global JGDAS_ENKF_ARCHIVE_TARS and JGDAS_ENKF_ARCHIVE_VRFY jobs and associated bash and python scripts Refs https://github.com/NOAA-EMC/global-workflow/issues/2451 # Type of change - Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO # How has this been tested? - Clone and build on RDHPCS - Cycled tests on Hercules - Forecast-only tests on Hercules # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [x] I have made corresponding changes to the documentation if necessary --- jobs/JGDAS_ENKF_ARCHIVE_TARS | 1 - jobs/JGDAS_ENKF_ARCHIVE_VRFY | 1 - scripts/exgdas_enkf_earc_tars.py | 2 +- scripts/exgdas_enkf_earc_vrfy.py | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/jobs/JGDAS_ENKF_ARCHIVE_TARS b/jobs/JGDAS_ENKF_ARCHIVE_TARS index 04fe8e3141..eec053bfa9 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE_TARS +++ b/jobs/JGDAS_ENKF_ARCHIVE_TARS @@ -7,7 +7,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_tars" -c "base earc_tars" ############################################## # Set variables used in the script ############################################## -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \ diff --git a/jobs/JGDAS_ENKF_ARCHIVE_VRFY b/jobs/JGDAS_ENKF_ARCHIVE_VRFY index 0094401987..e6e4f8d2d0 100755 --- a/jobs/JGDAS_ENKF_ARCHIVE_VRFY +++ b/jobs/JGDAS_ENKF_ARCHIVE_VRFY @@ -7,7 +7,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_vrfy" -c "base earc_vrfy" ############################################## # Set variables used in the script ############################################## -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \ COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \ diff --git a/scripts/exgdas_enkf_earc_tars.py b/scripts/exgdas_enkf_earc_tars.py index 63c978d6a8..3195fb41e5 100755 --- a/scripts/exgdas_enkf_earc_tars.py +++ b/scripts/exgdas_enkf_earc_tars.py @@ -38,7 +38,7 @@ def main(): # Also import all COMIN* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COM"): + if key.startswith("COMIN"): archive_dict[key] = archive.task_config[key] cwd = os.getcwd() diff --git a/scripts/exgdas_enkf_earc_vrfy.py b/scripts/exgdas_enkf_earc_vrfy.py index b7dd33a89a..3947885673 100755 --- a/scripts/exgdas_enkf_earc_vrfy.py +++ b/scripts/exgdas_enkf_earc_vrfy.py @@ -31,7 +31,7 @@ def main(): # Also import all COMIN* directory and template variables for key in archive.task_config.keys(): - if key.startswith("COM"): + if key.startswith("COMIN"): archive_dict[key] = archive.task_config[key] cwd = os.getcwd()