Merge branch 'bug/NCO-extractvars' of https://github.com/AntonMFernan…

…do-NOAA/global-workflow into bug/NCO-extractvars

ci/Jenkinsfile4AWS

@@ -6,7 +6,7 @@ def CI_CASES = ''
 def GH = 'none'
 // Location of the custom workspaces for each machine in the CI system.  They are persistent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow']
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: /lustre/jenkins/global-workflow/CI]
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins']
 def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git'
 def STATUS = 'Passed'
 

jobs/JGDAS_ENKF_ARCHIVE_TARS

@@ -7,7 +7,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_tars" -c "base earc_tars"
 ##############################################
 # Set variables used in the script
 ##############################################
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP
 MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
   COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
   COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \

jobs/JGDAS_ENKF_ARCHIVE_VRFY

@@ -7,7 +7,6 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_vrfy" -c "base earc_vrfy"
 ##############################################
 # Set variables used in the script
 ##############################################
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP
 MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
   COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
   COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \

scripts/exgdas_enkf_earc_tars.py

@@ -38,7 +38,7 @@ def main():
 
     # Also import all COMIN* directory and template variables
     for key in archive.task_config.keys():
-        if key.startswith("COM"):
+        if key.startswith("COMIN"):
             archive_dict[key] = archive.task_config[key]
 
     cwd = os.getcwd()

scripts/exgdas_enkf_earc_vrfy.py

@@ -31,7 +31,7 @@ def main():
 
     # Also import all COMIN* directory and template variables
     for key in archive.task_config.keys():
-        if key.startswith("COM"):
+        if key.startswith("COMIN"):
             archive_dict[key] = archive.task_config[key]
 
     cwd = os.getcwd()

ush/python/pygfs/task/marine_letkf.py

@@ -184,8 +184,26 @@ def finalize(self):
 
         letkfsaveconf = AttrDict()
         keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN', 'GDUMP_ENS',
-                'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL']
+                'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL',
+                'COMOUT_OCEAN_LETKF', 'COMOUT_ICE_LETKF', 'WINDOW_MIDDLE',
+                'MARINE_OBS_LIST_YAML']
         for key in keys:
             letkfsaveconf[key] = self.task_config[key]
+
+        # get the list of obs output files
+        obs_list = parse_j2yaml(letkfsaveconf.MARINE_OBS_LIST_YAML, self.task_config)
+        obs_files = []
+        for ob in obs_list['observers']:
+            obs_files.append(ob['obs space']['obsdataout']['engine']['obsfile'])
+        obs_files_to_copy = []
+        # copy obs from diags to COMOUT
+        for obs_src in obs_files:
+            obs_dst = os.path.join(letkfsaveconf.COMOUT_OCEAN_LETKF, 'diags',
+                                   os.path.basename(obs_src))
+            if os.path.exists(obs_src):
+                obs_files_to_copy.append([obs_src, obs_dst])
+        # stage the desired diag files
+        FileHandler({'mkdir': [os.path.join(letkfsaveconf.COMOUT_OCEAN_LETKF, 'diags')]}).sync()
+        FileHandler({'copy': obs_files_to_copy}).sync()
         letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf)
         FileHandler(letkf_save_list).sync()