From f071994c1e0f1a47cb92baf9575dcf4ee2e1c31a Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Tue, 25 Feb 2025 19:42:32 +0000 Subject: [PATCH 01/27] change noaacloud NodeName --- ci/Jenkinsfile4AWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index cac3f4cd11..c23ace3a96 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -5,7 +5,7 @@ def HOMEgfs = 'none' def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'AWS'] +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: /lustre/jenkins/global-workflow/CI] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' @@ -79,7 +79,7 @@ pipeline { Machine = machine[0].toUpperCase() + machine.substring(1) echo "Getting Common Workspace for ${Machine}" ws("${custom_workspace[machine]}/${env.CHANGE_ID}") { - properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea', 'AWS'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) + properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea', 'awsepicglobalworkflow'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])]) GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim() CUSTOM_WORKSPACE = "${WORKSPACE}" HOMEgfs = "${CUSTOM_WORKSPACE}/global-workflow" From a4771df691c38e0275f446fe295d1d00fe205538 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 26 Feb 2025 18:36:43 +0000 Subject: [PATCH 02/27] add download fix subset data, also serve as a test PR to trigger CI testing on AWS --- ush/fetch-fix-data.py | 331 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 ush/fetch-fix-data.py diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py new file mode 100644 index 0000000000..717fecdcf7 --- /dev/null +++ b/ush/fetch-fix-data.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +# cfetch-fix-data.py +# wei.huang@noaa.gov +# 2025-02-26 +# script to download a subset of FIX data to local machines. +import os +import time +import sys +#import requests +#import json +#import base64 +import getopt +import subprocess +from pathlib import Path + +#---------------------------------------------------------------------------------------------------------------- +class FetchFIXdata(): + def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): + self.aws_fix_bucket = 's3://noaa-nws-global-pds/fix' + self.aws_cp = 'aws --no-sign-request s3 cp' + self.aws_sync = 'aws --no-sign-request s3 sync' + + self.atmgridarray = atmgridarray + self.ocngridarray = ocngridarray + self.localdir = localdir + self.verbose = verbose + + #if (os.path.isdir(localdir)): + # print('Prepare to download FIX data for %s and %s to %s' %(atmgrid, ocngrid, localdir)) + #else: + # print('local dir: <%s> does not exist. Stop' %(localdir)) + # sys.exit(-1) + + self.verdict = {} + self.s3dict = {} + self.s3dict['raworog'] = 'raw/orog' + + if (self.localdir.find('fix') < 0): + self.targetdir = '%s/fix.subset' %(self.localdir) + else: + self.targetdir = self.localdir + +#---------------------------------------------------------------------------------------------------------------- + def update_s3dict(self): + self.update_s3dick_grid_independent() + self.add_grid_data() + + if (self.verbose): + self.printinfo() + +#---------------------------------------------------------------------------------------------------------------- + def update_s3dick_grid_independent(self): + for key in self.fix_ver_dict.keys(): + val = self.fix_ver_dict[key] + if (key == 'aer_ver'): + self.s3dict['aer'] = 'aer/%s' %(val) + elif (key == 'am_ver'): + self.s3dict['am'] = 'am/%s' %(val) + elif (key == 'chem_ver'): + self.s3dict['fimdata_chem'] = 'chem/%s/fimdata_chem' %(val) + self.s3dict['Emission_data'] = 'chem/%s/Emission_data' %(val) + elif (key == 'datm_ver'): + self.s3dict['cfsr'] = 'datm/%s/cfsr' %(val) + self.s3dict['gefs'] = 'datm/%s/gefs' %(val) + self.s3dict['gfs'] = 'datm/%s/gfs' %(val) + self.s3dict['mom6'] = 'datm/%s/mom6' %(val) + elif (key == 'glwu_ver'): + self.s3dict['glwu'] = 'glwu/%s' %(val) + elif (key == 'gsi_ver'): + self.s3dict['gsi'] = 'gsi/%s' %(val) + elif (key == 'lut_ver'): + self.s3dict['lut'] = 'lut/%s' %(val) + elif (key == 'mom6_ver'): + self.s3dict['mom6post'] = 'mom6/%s/post' %(val) + elif (key == 'reg2grb2_ver'): + self.s3dict['reg2grb2'] = 'reg2grb2/%s' %(val) + elif (key == 'sfc_climb_ver'): + self.s3dict['sfc_climo'] = 'sfc_climo/%s' %(val) + elif (key == 'verif_ver'): + self.s3dict['verif'] = 'verif/%s' %(val) + elif (key == 'wave_ver'): + self.s3dict['wave'] = 'wave/%s' %(val) + +#---------------------------------------------------------------------------------------------------------------- + def add_grid_data(self): + for key in self.fix_ver_dict.keys(): + val = self.fix_ver_dict[key] + if (key == 'orog_ver'): + self.add_atmgrid2s3dict('orog', key, val) + elif (key == 'ugwd_ver'): + self.add_atmgrid2s3dict('ugwd', key, val) + elif (key == 'mom6_ver'): + self.add_ocngrid2s3dict('mom6', key, val) + elif (key == 'cice_ver'): + self.add_ocngrid2s3dict('cice', key, val) + elif (key == 'cpl_ver'): + self.add_cpl2s3dict('cpl', key, val) + +#---------------------------------------------------------------------------------------------------------------- + def add_atmgrid2s3dict(self, varname, key, val): + for atmgrid in self.atmgridarray: + newkey = '%s_%s' %(key, atmgrid) + self.s3dict[newkey] = '%s/%s/%s' %(varname, val, atmgrid) + +#---------------------------------------------------------------------------------------------------------------- + def add_ocngrid2s3dict(self, varname, key, val): + for ocngrid in self.ocngridarray: + newkey = '%s_%s' %(key, atmgrid) + self.s3dict[newkey] = '%s/%s/%s' %(varname, val, ocngrid) + +#---------------------------------------------------------------------------------------------------------------- + def add_cpl2s3dict(self, varname, key, val): + for atmgrid in self.atmgridarray: + for ocngrid in self.ocngridarray: + newkey = '%s_a%so%s' %(key, atmgrid, ocngrid) + self.s3dict[newkey] = '%s/%s/a%so%s' %(varname, val, atmgrid, ocngrid) + +#---------------------------------------------------------------------------------------------------------------- + def printinfo(self): + print('Preparing to fetch') + print('ATM grid: ', self.atmgridarray) + print('ONC grid: ', self.ocngridarray) + print('From: %s' %(self.aws_fix_bucket)) + print('To: %s' %(self.targetdir)) + for key in self.s3dict.keys(): + val = self.s3dict[key] + print('%s: %s' %(key, val)) + +#---------------------------------------------------------------------------------------------------------------- + def fetchdata(self): + if (self.verbose): + print('Create local fix dir: ', self.targetdir) + + path = Path(self.targetdir) + path.mkdir(parents=True, exist_ok=True) + + self.fetch_ugwp_limb_tau() + + for key in self.s3dict.keys(): + self.fetch_dir(self.s3dict[key]) + +#---------------------------------------------------------------------------------------------------------------- + def fetch_dir(self, dir): + remotedir = '%s/%s' %(self.aws_fix_bucket, dir) + localdir = '%s/%s' %(self.targetdir, dir) + cmd = '%s %s %s'%(self.aws_sync, remotedir, localdir) + self.download_dir(cmd, localdir) + +#---------------------------------------------------------------------------------------------------------------- + def download_dir(self, cmd, localdir): + #returned_value = os.system(cmd) # returns the exit code in unix + #print('returned value:', returned_value) + + if (os.path.isdir(localdir)): + print('%s already exist. skip' %(localdir)) + else: + parentdir, dirname = os.path.split(localdir) + if (self.verbose): + print('Create local %s dir: ', parentdir) + path = Path(parentdir) + path.mkdir(parents=True, exist_ok=True) + if (self.verbose): + print(cmd) + print('Downloading ', localdir) + returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix + if (self.verbose): + print('returned value:', returned_value) + +#---------------------------------------------------------------------------------------------------------------- + def fetch_ugwp_limb_tau(self): + ugwp_limb_tau_remotepath = '%s/ugwd/%s/ugwp_limb_tau.nc' %(self.aws_fix_bucket, self.fix_ver_dict['ugwd_ver']) + ugwp_limb_tau_localdir = '%s/ugwd/%s' %(self.targetdir, self.fix_ver_dict['ugwd_ver']) + filename = '%s/ugwp_limb_tau.nc' %(ugwp_limb_tau_localdir) + path = Path(ugwp_limb_tau_localdir) + path.mkdir(parents=True, exist_ok=True) + cmd = '%s %s %s'%(self.aws_cp, ugwp_limb_tau_remotepath, filename) + self.download_file(cmd, filename) + +#---------------------------------------------------------------------------------------------------------------- + def download_file(self, cmd, filename): + #returned_value = os.system(cmd) # returns the exit code in unix + #print('returned value:', returned_value) + + if (os.path.isfile(filename)): + print('%s already exist. skip' %(filename)) + else: + if (self.verbose): + print(cmd) + print('Downloading ', filename) + returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix + if (self.verbose): + print('returned value:', returned_value) + +#---------------------------------------------------------------------------------------------------------------- + def set_fix_ver_from_gwhome(self, gwhome, verdict): + fix_ver_file = '%s/versions/fix.ver' + self.fix_ver_dict = verdict + if (os.path.isfile(fix_ver_file)): + with open(fix_ver_file, "r") as file: + for line in file.readlines(): + if (line.find('export ') >= 0): + headstr, _, value = line.strip().partition('=') + exphead, _, key = headstr.partition(' ') + self.fix_ver_dict[key] = value + else: + print('fix_ver_file: %s does not exist.' %(fix_ver_file)) + +#---------------------------------------------------------------------------------------------------------------- + def set_default_fix_ver(self, verdict): + self.fix_ver_dict = verdict + +#---------------------------------------------------------------------------------------------------------------- +def print_usage(verdict): + print('Usage: python fetch-fix-data.py \\') + print(' --atmgrid=AtmospericGrid (for multiple grids, separate with ",") \\') + print(' --ocngrid=OceanGrid (for multiple grids, separate with ",") \\') + print(' --localdir=Your-local-fix-dir \\') + print(' [options]') + print('options are:') + print('\t--gwhome=xxxx (Global-Workflow directory)') + + for key in verdict.keys(): + print('\t--%s=yyyymmdd default: %s' %(key, verdict[key])) + +#---------------------------------------------------------------------------------------------------------------- +if __name__ == '__main__': + atmgridlist = ['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] + ocngridlist = ['500', '100', '050', '025'] + + verbose = 0 + atmgrid = 'C48' + ocngrid = '500' + localdir = '/contrib/global-workflow-shared-data' + + #default fix-version + verdict = {} + verdict['aer_ver'] = '20220805' + verdict['am_ver'] = '20220805' + verdict['chem_ver'] = '20220805' + verdict['cice_ver'] = '20240416' + verdict['cpl_ver'] = '20230526' + verdict['datm_ver'] = '20220805' + verdict['glwu_ver'] = '20220805' + verdict['gsi_ver'] = '20240208' + verdict['lut_ver'] = '20220805' + verdict['mom6_ver'] = '20240416' + verdict['orog_ver'] = '20231027' + verdict['reg2grb2_ver'] = '20220805' + verdict['sfc_climo_ver'] = '20220805' + verdict['ugwd_ver'] = '20240624' + verdict['verif_ver'] = '20220805' + verdict['wave_ver'] = '20240105' + + gwhome=None + + opts, args = getopt.getopt(sys.argv[1:], '', ['help', 'atmgrid=', 'ocngrid=', + 'verbose=', 'localdir=', + 'gwhome=', + 'aer_ver=', + 'am_ver=', + 'chem_ver=', + 'cice_ver=', + 'cpl_ver=', + 'datm_ver=', + 'glwu_ver=', + 'gsi_ver=', + 'lut_ver=', + 'mom6_ver=', + 'orog_ver=', + 'reg2grb2_ver=', + 'sfc_climo_ver=', + 'ugwd_ver=', + 'verif_ver=', + 'wave_ver=']) + for o, a in opts: + print('o: %s, a: %s' %(o, a)) + if o in ['--help']: + print_usage(verdict) + sys.exit(0) + elif o in ['--verbose']: + verbose = int(a) + elif o in ['--atmgrid']: + atmgrid = a + elif o in ['--ocngrid']: + ocngrid = a + elif o in ['--localdir']: + localdir = a + elif o in ['--gwhome']: + gwhome = a + else: + _, vername = o.split('--') + print('vername: <%s>' %(vername)) + verdict[vername] = a + + if (atmgrid.find(',') > 0): + atmgridarray = atmgrid.split(',') + else: + atmgridarray = [atmgrid] + + for grid in atmgridarray: + if (grid not in atmgridlist): + print('atmgrid: ', grid) + print('is not in supported grids: ', atmgridlist) + print_usage(verdict) + sys.exit(-1) + + if (ocngrid.find(',') > 0): + ocngridarray = ocngrid.split(',') + else: + ocngridarray = [ocngrid] + + for grid in ocngridarray: + if (grid not in ocngridlist): + print('ocngrid: ', grid) + print('is not in supported grids: ', ocngridlist) + print_usage(verdict) + sys.exit(-1) + +#------------------------------------------------------------------ + ffd = FetchFIXdata(atmgridarray=atmgridarray, + ocngridarray=ocngridarray, + localdir=localdir, verbose=verbose) + + if (gwhome is None): + ffd.set_default_fix_ver(verdict) + else: + ffd.set_fix_ver_from_gwhome(gwhome, verdict) + + ffd.update_s3dict() + + ffd.fetchdata() From 4f1e67956a9718dee7cc5e0c784df55ca6f15d61 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 26 Feb 2025 19:26:17 +0000 Subject: [PATCH 03/27] fix pynorm error --- ush/fetch-fix-data.py | 126 +++++++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 56 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index 717fecdcf7..ce4fa1bdec 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -6,9 +6,6 @@ import os import time import sys -#import requests -#import json -#import base64 import getopt import subprocess from pathlib import Path @@ -16,6 +13,7 @@ #---------------------------------------------------------------------------------------------------------------- class FetchFIXdata(): def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): + self.aws_fix_bucket = 's3://noaa-nws-global-pds/fix' self.aws_cp = 'aws --no-sign-request s3 cp' self.aws_sync = 'aws --no-sign-request s3 sync' @@ -25,11 +23,11 @@ def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, ve self.localdir = localdir self.verbose = verbose - #if (os.path.isdir(localdir)): - # print('Prepare to download FIX data for %s and %s to %s' %(atmgrid, ocngrid, localdir)) - #else: - # print('local dir: <%s> does not exist. Stop' %(localdir)) - # sys.exit(-1) + #if (os.path.isdir(localdir)): + # print('Prepare to download FIX data for %s and %s to %s' %(atmgrid, ocngrid, localdir)) + #else: + # print('local dir: <%s> does not exist. Stop' %(localdir)) + # sys.exit(-1) self.verdict = {} self.s3dict = {} @@ -40,49 +38,52 @@ def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, ve else: self.targetdir = self.localdir -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def update_s3dict(self): + self.update_s3dick_grid_independent() self.add_grid_data() if (self.verbose): self.printinfo() -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def update_s3dick_grid_independent(self): + for key in self.fix_ver_dict.keys(): val = self.fix_ver_dict[key] if (key == 'aer_ver'): - self.s3dict['aer'] = 'aer/%s' %(val) + self.s3dict['aer'] = 'aer/%s' %( val ) elif (key == 'am_ver'): - self.s3dict['am'] = 'am/%s' %(val) + self.s3dict['am'] = 'am/%s' %( val ) elif (key == 'chem_ver'): - self.s3dict['fimdata_chem'] = 'chem/%s/fimdata_chem' %(val) - self.s3dict['Emission_data'] = 'chem/%s/Emission_data' %(val) + self.s3dict['fimdata_chem'] = 'chem/%s/fimdata_chem' %( val ) + self.s3dict['Emission_data'] = 'chem/%s/Emission_data' %( val ) elif (key == 'datm_ver'): - self.s3dict['cfsr'] = 'datm/%s/cfsr' %(val) - self.s3dict['gefs'] = 'datm/%s/gefs' %(val) - self.s3dict['gfs'] = 'datm/%s/gfs' %(val) - self.s3dict['mom6'] = 'datm/%s/mom6' %(val) + self.s3dict['cfsr'] = 'datm/%s/cfsr' %( val ) + self.s3dict['gefs'] = 'datm/%s/gefs' %( val ) + self.s3dict['gfs'] = 'datm/%s/gfs' %( val ) + self.s3dict['mom6'] = 'datm/%s/mom6' %( val ) elif (key == 'glwu_ver'): - self.s3dict['glwu'] = 'glwu/%s' %(val) + self.s3dict['glwu'] = 'glwu/%s' %( val ) elif (key == 'gsi_ver'): - self.s3dict['gsi'] = 'gsi/%s' %(val) + self.s3dict['gsi'] = 'gsi/%s' %( val ) elif (key == 'lut_ver'): - self.s3dict['lut'] = 'lut/%s' %(val) + self.s3dict['lut'] = 'lut/%s' %( val ) elif (key == 'mom6_ver'): - self.s3dict['mom6post'] = 'mom6/%s/post' %(val) + self.s3dict['mom6post'] = 'mom6/%s/post' %( val ) elif (key == 'reg2grb2_ver'): - self.s3dict['reg2grb2'] = 'reg2grb2/%s' %(val) + self.s3dict['reg2grb2'] = 'reg2grb2/%s' %( val ) elif (key == 'sfc_climb_ver'): - self.s3dict['sfc_climo'] = 'sfc_climo/%s' %(val) + self.s3dict['sfc_climo'] = 'sfc_climo/%s' %( val ) elif (key == 'verif_ver'): - self.s3dict['verif'] = 'verif/%s' %(val) + self.s3dict['verif'] = 'verif/%s' %( val ) elif (key == 'wave_ver'): - self.s3dict['wave'] = 'wave/%s' %(val) + self.s3dict['wave'] = 'wave/%s' %( val ) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def add_grid_data(self): + for key in self.fix_ver_dict.keys(): val = self.fix_ver_dict[key] if (key == 'orog_ver'): @@ -96,38 +97,43 @@ def add_grid_data(self): elif (key == 'cpl_ver'): self.add_cpl2s3dict('cpl', key, val) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def add_atmgrid2s3dict(self, varname, key, val): + for atmgrid in self.atmgridarray: - newkey = '%s_%s' %(key, atmgrid) - self.s3dict[newkey] = '%s/%s/%s' %(varname, val, atmgrid) + newkey = '%s_%s' %( key, atmgrid ) + self.s3dict[newkey] = '%s/%s/%s' %( varname, val, atmgrid ) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def add_ocngrid2s3dict(self, varname, key, val): + for ocngrid in self.ocngridarray: - newkey = '%s_%s' %(key, atmgrid) - self.s3dict[newkey] = '%s/%s/%s' %(varname, val, ocngrid) + newkey = '%s_%s' %( key, atmgrid ) + self.s3dict[newkey] = '%s/%s/%s' %( varname, val, ocngrid ) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def add_cpl2s3dict(self, varname, key, val): + for atmgrid in self.atmgridarray: for ocngrid in self.ocngridarray: - newkey = '%s_a%so%s' %(key, atmgrid, ocngrid) - self.s3dict[newkey] = '%s/%s/a%so%s' %(varname, val, atmgrid, ocngrid) + newkey = '%s_a%so%s' %( key, atmgrid, ocngrid ) + self.s3dict[newkey] = '%s/%s/a%so%s' %( varname, val, atmgrid, ocngrid ) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def printinfo(self): + print('Preparing to fetch') print('ATM grid: ', self.atmgridarray) print('ONC grid: ', self.ocngridarray) - print('From: %s' %(self.aws_fix_bucket)) - print('To: %s' %(self.targetdir)) + print('From: %s' %( self.aws_fix_bucket )) + print('To: %s' %( self.targetdir )) for key in self.s3dict.keys(): val = self.s3dict[key] - print('%s: %s' %(key, val)) + print('%s: %s' %( key, val )) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def fetchdata(self): + if (self.verbose): print('Create local fix dir: ', self.targetdir) @@ -139,17 +145,19 @@ def fetchdata(self): for key in self.s3dict.keys(): self.fetch_dir(self.s3dict[key]) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def fetch_dir(self, dir): + remotedir = '%s/%s' %(self.aws_fix_bucket, dir) - localdir = '%s/%s' %(self.targetdir, dir) + localdir = '%s/%s' %( self.targetdir, dir ) cmd = '%s %s %s'%(self.aws_sync, remotedir, localdir) self.download_dir(cmd, localdir) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def download_dir(self, cmd, localdir): - #returned_value = os.system(cmd) # returns the exit code in unix - #print('returned value:', returned_value) + + #returned_value = os.system(cmd) # returns the exit code in unix + #print('returned value:', returned_value) if (os.path.isdir(localdir)): print('%s already exist. skip' %(localdir)) @@ -166,8 +174,9 @@ def download_dir(self, cmd, localdir): if (self.verbose): print('returned value:', returned_value) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def fetch_ugwp_limb_tau(self): + ugwp_limb_tau_remotepath = '%s/ugwd/%s/ugwp_limb_tau.nc' %(self.aws_fix_bucket, self.fix_ver_dict['ugwd_ver']) ugwp_limb_tau_localdir = '%s/ugwd/%s' %(self.targetdir, self.fix_ver_dict['ugwd_ver']) filename = '%s/ugwp_limb_tau.nc' %(ugwp_limb_tau_localdir) @@ -176,13 +185,14 @@ def fetch_ugwp_limb_tau(self): cmd = '%s %s %s'%(self.aws_cp, ugwp_limb_tau_remotepath, filename) self.download_file(cmd, filename) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def download_file(self, cmd, filename): + #returned_value = os.system(cmd) # returns the exit code in unix #print('returned value:', returned_value) if (os.path.isfile(filename)): - print('%s already exist. skip' %(filename)) + print('%s already exist. skip' %( filename )) else: if (self.verbose): print(cmd) @@ -191,8 +201,9 @@ def download_file(self, cmd, filename): if (self.verbose): print('returned value:', returned_value) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def set_fix_ver_from_gwhome(self, gwhome, verdict): + fix_ver_file = '%s/versions/fix.ver' self.fix_ver_dict = verdict if (os.path.isfile(fix_ver_file)): @@ -205,12 +216,14 @@ def set_fix_ver_from_gwhome(self, gwhome, verdict): else: print('fix_ver_file: %s does not exist.' %(fix_ver_file)) -#---------------------------------------------------------------------------------------------------------------- + #---------------------------------------------------------------------------------------------------------------- def set_default_fix_ver(self, verdict): + self.fix_ver_dict = verdict #---------------------------------------------------------------------------------------------------------------- def print_usage(verdict): + print('Usage: python fetch-fix-data.py \\') print(' --atmgrid=AtmospericGrid (for multiple grids, separate with ",") \\') print(' --ocngrid=OceanGrid (for multiple grids, separate with ",") \\') @@ -220,10 +233,11 @@ def print_usage(verdict): print('\t--gwhome=xxxx (Global-Workflow directory)') for key in verdict.keys(): - print('\t--%s=yyyymmdd default: %s' %(key, verdict[key])) + print( '\t--%s=yyyymmdd default: %s' %( key, verdict[key] ) ) #---------------------------------------------------------------------------------------------------------------- if __name__ == '__main__': + atmgridlist = ['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] ocngridlist = ['500', '100', '050', '025'] @@ -232,7 +246,7 @@ def print_usage(verdict): ocngrid = '500' localdir = '/contrib/global-workflow-shared-data' - #default fix-version + #default fix-version verdict = {} verdict['aer_ver'] = '20220805' verdict['am_ver'] = '20220805' @@ -273,7 +287,7 @@ def print_usage(verdict): 'verif_ver=', 'wave_ver=']) for o, a in opts: - print('o: %s, a: %s' %(o, a)) + #print( 'o: %s, a: %s' %(o, a) ) if o in ['--help']: print_usage(verdict) sys.exit(0) @@ -289,7 +303,7 @@ def print_usage(verdict): gwhome = a else: _, vername = o.split('--') - print('vername: <%s>' %(vername)) + print( 'vername: <%s>' %(vername) ) verdict[vername] = a if (atmgrid.find(',') > 0): @@ -316,7 +330,7 @@ def print_usage(verdict): print_usage(verdict) sys.exit(-1) -#------------------------------------------------------------------ + #------------------------------------------------------------------ ffd = FetchFIXdata(atmgridarray=atmgridarray, ocngridarray=ocngridarray, localdir=localdir, verbose=verbose) From 2241227b7319fc5dfdc6b866fbef281dc0144a65 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 26 Feb 2025 20:10:17 +0000 Subject: [PATCH 04/27] fix pynorm error 2 --- ush/fetch-fix-data.py | 107 +++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index ce4fa1bdec..2a39f0d60d 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -10,8 +10,9 @@ import subprocess from pathlib import Path -#---------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------- class FetchFIXdata(): + def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): self.aws_fix_bucket = 's3://noaa-nws-global-pds/fix' @@ -38,7 +39,7 @@ def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, ve else: self.targetdir = self.localdir - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def update_s3dict(self): self.update_s3dick_grid_independent() @@ -47,41 +48,41 @@ def update_s3dict(self): if (self.verbose): self.printinfo() - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def update_s3dick_grid_independent(self): for key in self.fix_ver_dict.keys(): val = self.fix_ver_dict[key] - if (key == 'aer_ver'): - self.s3dict['aer'] = 'aer/%s' %( val ) - elif (key == 'am_ver'): - self.s3dict['am'] = 'am/%s' %( val ) - elif (key == 'chem_ver'): - self.s3dict['fimdata_chem'] = 'chem/%s/fimdata_chem' %( val ) - self.s3dict['Emission_data'] = 'chem/%s/Emission_data' %( val ) - elif (key == 'datm_ver'): - self.s3dict['cfsr'] = 'datm/%s/cfsr' %( val ) - self.s3dict['gefs'] = 'datm/%s/gefs' %( val ) - self.s3dict['gfs'] = 'datm/%s/gfs' %( val ) - self.s3dict['mom6'] = 'datm/%s/mom6' %( val ) - elif (key == 'glwu_ver'): - self.s3dict['glwu'] = 'glwu/%s' %( val ) - elif (key == 'gsi_ver'): - self.s3dict['gsi'] = 'gsi/%s' %( val ) - elif (key == 'lut_ver'): - self.s3dict['lut'] = 'lut/%s' %( val ) - elif (key == 'mom6_ver'): - self.s3dict['mom6post'] = 'mom6/%s/post' %( val ) - elif (key == 'reg2grb2_ver'): - self.s3dict['reg2grb2'] = 'reg2grb2/%s' %( val ) - elif (key == 'sfc_climb_ver'): - self.s3dict['sfc_climo'] = 'sfc_climo/%s' %( val ) - elif (key == 'verif_ver'): - self.s3dict['verif'] = 'verif/%s' %( val ) - elif (key == 'wave_ver'): - self.s3dict['wave'] = 'wave/%s' %( val ) - - #---------------------------------------------------------------------------------------------------------------- + if ( key == 'aer_ver' ): + self.s3dict['aer'] = 'aer/%s' %(val) + elif ( key == 'am_ver' ): + self.s3dict['am'] = 'am/%s' %(val) + elif ( key == 'chem_ver' ): + self.s3dict['fimdata_chem'] = 'chem/%s/fimdata_chem' %(val) + self.s3dict['Emission_data'] = 'chem/%s/Emission_data' %(val) + elif ( key == 'datm_ver' ): + self.s3dict['cfsr'] = 'datm/%s/cfsr' %(val) + self.s3dict['gefs'] = 'datm/%s/gefs' %(val) + self.s3dict['gfs'] = 'datm/%s/gfs' %(val) + self.s3dict['mom6'] = 'datm/%s/mom6' %(val) + elif ( key == 'glwu_ver' ): + self.s3dict['glwu'] = 'glwu/%s' %(val) + elif ( key == 'gsi_ver' ): + self.s3dict['gsi'] = 'gsi/%s' %(val) + elif ( key == 'lut_ver' ): + self.s3dict['lut'] = 'lut/%s' %(val) + elif ( key == 'mom6_ver' ): + self.s3dict['mom6post'] = 'mom6/%s/post' %(val) + elif ( key == 'reg2grb2_ver' ): + self.s3dict['reg2grb2'] = 'reg2grb2/%s' %(val) + elif ( key == 'sfc_climb_ver' ): + self.s3dict['sfc_climo'] = 'sfc_climo/%s' %(val) + elif ( key == 'verif_ver' ): + self.s3dict['verif'] = 'verif/%s' %(val) + elif ( key == 'wave_ver' ): + self.s3dict['wave'] = 'wave/%s' %(val) + + # ---------------------------------------------------------------------------------------------------------------- def add_grid_data(self): for key in self.fix_ver_dict.keys(): @@ -97,29 +98,29 @@ def add_grid_data(self): elif (key == 'cpl_ver'): self.add_cpl2s3dict('cpl', key, val) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def add_atmgrid2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: newkey = '%s_%s' %( key, atmgrid ) - self.s3dict[newkey] = '%s/%s/%s' %( varname, val, atmgrid ) + self.s3dict[newkey] = '%s/%s/%s' %(varname, val, atmgrid) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def add_ocngrid2s3dict(self, varname, key, val): for ocngrid in self.ocngridarray: - newkey = '%s_%s' %( key, atmgrid ) - self.s3dict[newkey] = '%s/%s/%s' %( varname, val, ocngrid ) + newkey = '%s_%s' %(key, atmgrid) + self.s3dict[newkey] = '%s/%s/%s' %(varname, val, ocngrid) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def add_cpl2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: for ocngrid in self.ocngridarray: newkey = '%s_a%so%s' %( key, atmgrid, ocngrid ) - self.s3dict[newkey] = '%s/%s/a%so%s' %( varname, val, atmgrid, ocngrid ) + self.s3dict[newkey] = '%s/%s/a%so%s' %(varname, val, atmgrid, ocngrid) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def printinfo(self): print('Preparing to fetch') @@ -131,7 +132,7 @@ def printinfo(self): val = self.s3dict[key] print('%s: %s' %( key, val )) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def fetchdata(self): if (self.verbose): @@ -145,15 +146,15 @@ def fetchdata(self): for key in self.s3dict.keys(): self.fetch_dir(self.s3dict[key]) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def fetch_dir(self, dir): remotedir = '%s/%s' %(self.aws_fix_bucket, dir) localdir = '%s/%s' %( self.targetdir, dir ) - cmd = '%s %s %s'%(self.aws_sync, remotedir, localdir) + cmd = '%s %s %s' %(self.aws_sync, remotedir, localdir) self.download_dir(cmd, localdir) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def download_dir(self, cmd, localdir): #returned_value = os.system(cmd) # returns the exit code in unix @@ -164,7 +165,7 @@ def download_dir(self, cmd, localdir): else: parentdir, dirname = os.path.split(localdir) if (self.verbose): - print('Create local %s dir: ', parentdir) + print('Create local %s dir: ' %(parentdir)) path = Path(parentdir) path.mkdir(parents=True, exist_ok=True) if (self.verbose): @@ -174,7 +175,7 @@ def download_dir(self, cmd, localdir): if (self.verbose): print('returned value:', returned_value) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def fetch_ugwp_limb_tau(self): ugwp_limb_tau_remotepath = '%s/ugwd/%s/ugwp_limb_tau.nc' %(self.aws_fix_bucket, self.fix_ver_dict['ugwd_ver']) @@ -185,7 +186,7 @@ def fetch_ugwp_limb_tau(self): cmd = '%s %s %s'%(self.aws_cp, ugwp_limb_tau_remotepath, filename) self.download_file(cmd, filename) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def download_file(self, cmd, filename): #returned_value = os.system(cmd) # returns the exit code in unix @@ -201,7 +202,7 @@ def download_file(self, cmd, filename): if (self.verbose): print('returned value:', returned_value) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def set_fix_ver_from_gwhome(self, gwhome, verdict): fix_ver_file = '%s/versions/fix.ver' @@ -216,12 +217,12 @@ def set_fix_ver_from_gwhome(self, gwhome, verdict): else: print('fix_ver_file: %s does not exist.' %(fix_ver_file)) - #---------------------------------------------------------------------------------------------------------------- + # ---------------------------------------------------------------------------------------------------------------- def set_default_fix_ver(self, verdict): self.fix_ver_dict = verdict -#---------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------- def print_usage(verdict): print('Usage: python fetch-fix-data.py \\') @@ -235,7 +236,7 @@ def print_usage(verdict): for key in verdict.keys(): print( '\t--%s=yyyymmdd default: %s' %( key, verdict[key] ) ) -#---------------------------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------------------------- if __name__ == '__main__': atmgridlist = ['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] @@ -330,7 +331,7 @@ def print_usage(verdict): print_usage(verdict) sys.exit(-1) - #------------------------------------------------------------------ + # ------------------------------------------------------------------ ffd = FetchFIXdata(atmgridarray=atmgridarray, ocngridarray=ocngridarray, localdir=localdir, verbose=verbose) From 9679617e8c7f9bc2978e7b66036fefe9e44f282b Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 26 Feb 2025 21:06:38 +0000 Subject: [PATCH 05/27] fix pynorm error3 --- ush/fetch-fix-data.py | 64 +++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index 2a39f0d60d..558f12bee9 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -24,10 +24,10 @@ def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, ve self.localdir = localdir self.verbose = verbose - #if (os.path.isdir(localdir)): + # if (os.path.isdir(localdir)): # print('Prepare to download FIX data for %s and %s to %s' %(atmgrid, ocngrid, localdir)) - #else: - # print('local dir: <%s> does not exist. Stop' %(localdir)) + # else: + # print(f'local dir: <{localdir}> does not exist. Stop') # sys.exit(-1) self.verdict = {} @@ -35,7 +35,7 @@ def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, ve self.s3dict['raworog'] = 'raw/orog' if (self.localdir.find('fix') < 0): - self.targetdir = '%s/fix.subset' %(self.localdir) + self.targetdir = f'{self.localdir}/fix.subset' else: self.targetdir = self.localdir @@ -53,34 +53,34 @@ def update_s3dick_grid_independent(self): for key in self.fix_ver_dict.keys(): val = self.fix_ver_dict[key] - if ( key == 'aer_ver' ): - self.s3dict['aer'] = 'aer/%s' %(val) - elif ( key == 'am_ver' ): - self.s3dict['am'] = 'am/%s' %(val) - elif ( key == 'chem_ver' ): - self.s3dict['fimdata_chem'] = 'chem/%s/fimdata_chem' %(val) - self.s3dict['Emission_data'] = 'chem/%s/Emission_data' %(val) - elif ( key == 'datm_ver' ): - self.s3dict['cfsr'] = 'datm/%s/cfsr' %(val) - self.s3dict['gefs'] = 'datm/%s/gefs' %(val) - self.s3dict['gfs'] = 'datm/%s/gfs' %(val) - self.s3dict['mom6'] = 'datm/%s/mom6' %(val) - elif ( key == 'glwu_ver' ): - self.s3dict['glwu'] = 'glwu/%s' %(val) - elif ( key == 'gsi_ver' ): - self.s3dict['gsi'] = 'gsi/%s' %(val) - elif ( key == 'lut_ver' ): - self.s3dict['lut'] = 'lut/%s' %(val) - elif ( key == 'mom6_ver' ): - self.s3dict['mom6post'] = 'mom6/%s/post' %(val) - elif ( key == 'reg2grb2_ver' ): - self.s3dict['reg2grb2'] = 'reg2grb2/%s' %(val) - elif ( key == 'sfc_climb_ver' ): - self.s3dict['sfc_climo'] = 'sfc_climo/%s' %(val) - elif ( key == 'verif_ver' ): - self.s3dict['verif'] = 'verif/%s' %(val) - elif ( key == 'wave_ver' ): - self.s3dict['wave'] = 'wave/%s' %(val) + if (key == 'aer_ver'): + self.s3dict['aer'] = f'aer/{val}' + elif ( key == 'am_ver'): + self.s3dict['am'] = f'am/{val}' + elif (key == 'chem_ver'): + self.s3dict['fimdata_chem'] = f'chem/{val}/fimdata_chem' + self.s3dict['Emission_data'] = f'chem/{val}/Emission_data' + elif (key == 'datm_ver'): + self.s3dict['cfsr'] = f'datm/{val}/cfsr' + self.s3dict['gefs'] = f'datm/{val}/gefs' + self.s3dict['gfs'] = f'datm/{val}/gfs' + self.s3dict['mom6'] = f'datm/{val}/mom6' + elif (key == 'glwu_ver'): + self.s3dict['glwu'] = f'glwu/{val}' + elif (key == 'gsi_ver'): + self.s3dict['gsi'] = f'gsi/{val}' + elif (key == 'lut_ver'): + self.s3dict['lut'] = f'lut/{val}' + elif (key == 'mom6_ver'): + self.s3dict['mom6post'] = f'mom6/{val}/post' + elif (key == 'reg2grb2_ver'): + self.s3dict['reg2grb2'] = f'reg2grb2/{val}' + elif (key == 'sfc_climb_ver' ): + self.s3dict['sfc_climo'] = f'sfc_climo/{val}' + elif (key == 'verif_ver'): + self.s3dict['verif'] = f'verif/{val}' + elif (key == 'wave_ver'): + self.s3dict['wave'] = f'wave/{val}' # ---------------------------------------------------------------------------------------------------------------- def add_grid_data(self): From fe887515dc8b8b3bed8312d189fb3ceddbe9ed85 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 26 Feb 2025 21:31:00 +0000 Subject: [PATCH 06/27] fix pynorm error 4 --- ush/fetch-fix-data.py | 74 ++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index 558f12bee9..fe7120285e 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -102,41 +102,41 @@ def add_grid_data(self): def add_atmgrid2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: - newkey = '%s_%s' %( key, atmgrid ) - self.s3dict[newkey] = '%s/%s/%s' %(varname, val, atmgrid) + newkey = f'{key}_{atmgrid}' + self.s3dict[newkey] = f'{varname}/{val}/{atmgrid}' # ---------------------------------------------------------------------------------------------------------------- def add_ocngrid2s3dict(self, varname, key, val): for ocngrid in self.ocngridarray: - newkey = '%s_%s' %(key, atmgrid) - self.s3dict[newkey] = '%s/%s/%s' %(varname, val, ocngrid) + newkey = f'{key}_{atmgrid}' + self.s3dict[newkey] = f'{varname}/{val}/{ocngrid}' # ---------------------------------------------------------------------------------------------------------------- def add_cpl2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: for ocngrid in self.ocngridarray: - newkey = '%s_a%so%s' %( key, atmgrid, ocngrid ) - self.s3dict[newkey] = '%s/%s/a%so%s' %(varname, val, atmgrid, ocngrid) + newkey = f'{key}_a{atmgrid}o{ocngrid}' + self.s3dict[newkey] = f'{varname}/{val}/a{atmgrid}o{ocngrid}' # ---------------------------------------------------------------------------------------------------------------- def printinfo(self): - print('Preparing to fetch') - print('ATM grid: ', self.atmgridarray) - print('ONC grid: ', self.ocngridarray) - print('From: %s' %( self.aws_fix_bucket )) - print('To: %s' %( self.targetdir )) + print(f'Preparing to fetch') + print(f'ATM grid: {self.atmgridarray}') + print(f'ONC grid: {self.ocngridarray}') + print(f'From: {self.aws_fix_bucket}') + print(f'To: {self.targetdir}') for key in self.s3dict.keys(): val = self.s3dict[key] - print('%s: %s' %( key, val )) + print(f'{key}: {val}') # ---------------------------------------------------------------------------------------------------------------- def fetchdata(self): if (self.verbose): - print('Create local fix dir: ', self.targetdir) + print('Create local fix dir: {self.targetdir}') path = Path(self.targetdir) path.mkdir(parents=True, exist_ok=True) @@ -149,28 +149,28 @@ def fetchdata(self): # ---------------------------------------------------------------------------------------------------------------- def fetch_dir(self, dir): - remotedir = '%s/%s' %(self.aws_fix_bucket, dir) - localdir = '%s/%s' %( self.targetdir, dir ) - cmd = '%s %s %s' %(self.aws_sync, remotedir, localdir) + remotedir = f'{self.aws_fix_bucket}/{dir}' + localdir = f'{self.targetdir}/{dir}' + cmd = f'{self.aws_sync} {remotedir} {localdir}' self.download_dir(cmd, localdir) # ---------------------------------------------------------------------------------------------------------------- def download_dir(self, cmd, localdir): - #returned_value = os.system(cmd) # returns the exit code in unix - #print('returned value:', returned_value) + # returned_value = os.system(cmd) # returns the exit code in unix + # print('returned value:', returned_value) if (os.path.isdir(localdir)): - print('%s already exist. skip' %(localdir)) + print(f'{localdir} already exist. skip' else: parentdir, dirname = os.path.split(localdir) if (self.verbose): - print('Create local %s dir: ' %(parentdir)) + print(f'Create local {parentdir} dir:') path = Path(parentdir) path.mkdir(parents=True, exist_ok=True) if (self.verbose): print(cmd) - print('Downloading ', localdir) + print(f'Downloading {localdir}') returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix if (self.verbose): print('returned value:', returned_value) @@ -178,26 +178,26 @@ def download_dir(self, cmd, localdir): # ---------------------------------------------------------------------------------------------------------------- def fetch_ugwp_limb_tau(self): - ugwp_limb_tau_remotepath = '%s/ugwd/%s/ugwp_limb_tau.nc' %(self.aws_fix_bucket, self.fix_ver_dict['ugwd_ver']) - ugwp_limb_tau_localdir = '%s/ugwd/%s' %(self.targetdir, self.fix_ver_dict['ugwd_ver']) - filename = '%s/ugwp_limb_tau.nc' %(ugwp_limb_tau_localdir) + ugwp_limb_tau_remotepath = f'{self.aws_fix_bucket}/ugwd/{self.fix_ver_dict['ugwd_ver']}/ugwp_limb_tau.nc' + ugwp_limb_tau_localdir = f'{self.targetdir}/ugwd/{self.fix_ver_dict['ugwd_ver']}' + filename = f'{ugwp_limb_tau_localdir}/ugwp_limb_tau.nc' path = Path(ugwp_limb_tau_localdir) path.mkdir(parents=True, exist_ok=True) - cmd = '%s %s %s'%(self.aws_cp, ugwp_limb_tau_remotepath, filename) + cmd = f'{self.aws_cp} {ugwp_limb_tau_remotepath} {filename}' self.download_file(cmd, filename) # ---------------------------------------------------------------------------------------------------------------- def download_file(self, cmd, filename): - #returned_value = os.system(cmd) # returns the exit code in unix - #print('returned value:', returned_value) + # returned_value = os.system(cmd) # returns the exit code in unix + # print('returned value:', returned_value) if (os.path.isfile(filename)): - print('%s already exist. skip' %( filename )) + print(f'{filename} already exist. skip') else: if (self.verbose): print(cmd) - print('Downloading ', filename) + print(f'Downloading {filename}') returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix if (self.verbose): print('returned value:', returned_value) @@ -205,7 +205,7 @@ def download_file(self, cmd, filename): # ---------------------------------------------------------------------------------------------------------------- def set_fix_ver_from_gwhome(self, gwhome, verdict): - fix_ver_file = '%s/versions/fix.ver' + fix_ver_file = f'{gwhome}/versions/fix.ver' self.fix_ver_dict = verdict if (os.path.isfile(fix_ver_file)): with open(fix_ver_file, "r") as file: @@ -215,7 +215,7 @@ def set_fix_ver_from_gwhome(self, gwhome, verdict): exphead, _, key = headstr.partition(' ') self.fix_ver_dict[key] = value else: - print('fix_ver_file: %s does not exist.' %(fix_ver_file)) + print(f'fix_ver_file: {ix_ver_file}s does not exist.') # ---------------------------------------------------------------------------------------------------------------- def set_default_fix_ver(self, verdict): @@ -225,6 +225,7 @@ def set_default_fix_ver(self, verdict): # ---------------------------------------------------------------------------------------------------------------- def print_usage(verdict): + print('Usage: python fetch-fix-data.py \\') print(' --atmgrid=AtmospericGrid (for multiple grids, separate with ",") \\') print(' --ocngrid=OceanGrid (for multiple grids, separate with ",") \\') @@ -234,11 +235,12 @@ def print_usage(verdict): print('\t--gwhome=xxxx (Global-Workflow directory)') for key in verdict.keys(): - print( '\t--%s=yyyymmdd default: %s' %( key, verdict[key] ) ) + print(f'\t--{key}=yyyymmdd default: {verdict[key]}') # ---------------------------------------------------------------------------------------------------------------- if __name__ == '__main__': + atmgridlist = ['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] ocngridlist = ['500', '100', '050', '025'] @@ -247,7 +249,7 @@ def print_usage(verdict): ocngrid = '500' localdir = '/contrib/global-workflow-shared-data' - #default fix-version + # default fix-version verdict = {} verdict['aer_ver'] = '20220805' verdict['am_ver'] = '20220805' @@ -266,7 +268,7 @@ def print_usage(verdict): verdict['verif_ver'] = '20220805' verdict['wave_ver'] = '20240105' - gwhome=None + gwhome = None opts, args = getopt.getopt(sys.argv[1:], '', ['help', 'atmgrid=', 'ocngrid=', 'verbose=', 'localdir=', @@ -288,7 +290,7 @@ def print_usage(verdict): 'verif_ver=', 'wave_ver=']) for o, a in opts: - #print( 'o: %s, a: %s' %(o, a) ) + # print(f'o: {o}, a: {a}') if o in ['--help']: print_usage(verdict) sys.exit(0) @@ -304,7 +306,7 @@ def print_usage(verdict): gwhome = a else: _, vername = o.split('--') - print( 'vername: <%s>' %(vername) ) + print(f'vername: <{vername}>') verdict[vername] = a if (atmgrid.find(',') > 0): From 0284fb7f3128c750d8df0b05400d3d754491246e Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 26 Feb 2025 23:00:23 +0000 Subject: [PATCH 07/27] fix pynorm error 5 --- ush/fetch-fix-data.py | 240 +++++++++++++++++++++--------------------- 1 file changed, 121 insertions(+), 119 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index fe7120285e..6d6f8ac4fe 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -13,16 +13,17 @@ # ---------------------------------------------------------------------------------------------------------------- class FetchFIXdata(): + def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): - self.aws_fix_bucket = 's3://noaa-nws-global-pds/fix' - self.aws_cp = 'aws --no-sign-request s3 cp' - self.aws_sync = 'aws --no-sign-request s3 sync' + self.aws_fix_bucket='s3://noaa-nws-global-pds/fix' + self.aws_cp='aws --no-sign-request s3 cp' + self.aws_sync='aws --no-sign-request s3 sync' - self.atmgridarray = atmgridarray - self.ocngridarray = ocngridarray - self.localdir = localdir - self.verbose = verbose + self.atmgridarray=atmgridarray + self.ocngridarray=ocngridarray + self.localdir=localdir + self.verbose=verbose # if (os.path.isdir(localdir)): # print('Prepare to download FIX data for %s and %s to %s' %(atmgrid, ocngrid, localdir)) @@ -30,14 +31,14 @@ def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, ve # print(f'local dir: <{localdir}> does not exist. Stop') # sys.exit(-1) - self.verdict = {} - self.s3dict = {} - self.s3dict['raworog'] = 'raw/orog' + self.verdict={} + self.s3dict={} + self.s3dict['raworog']='raw/orog' if (self.localdir.find('fix') < 0): - self.targetdir = f'{self.localdir}/fix.subset' + self.targetdir=f'{self.localdir}/fix.subset' else: - self.targetdir = self.localdir + self.targetdir=self.localdir # ---------------------------------------------------------------------------------------------------------------- def update_s3dict(self): @@ -52,41 +53,41 @@ def update_s3dict(self): def update_s3dick_grid_independent(self): for key in self.fix_ver_dict.keys(): - val = self.fix_ver_dict[key] + val=self.fix_ver_dict[key] if (key == 'aer_ver'): - self.s3dict['aer'] = f'aer/{val}' + self.s3dict['aer']=f'aer/{val}' elif ( key == 'am_ver'): - self.s3dict['am'] = f'am/{val}' + self.s3dict['am']=f'am/{val}' elif (key == 'chem_ver'): - self.s3dict['fimdata_chem'] = f'chem/{val}/fimdata_chem' - self.s3dict['Emission_data'] = f'chem/{val}/Emission_data' + self.s3dict['fimdata_chem']=f'chem/{val}/fimdata_chem' + self.s3dict['Emission_data']=f'chem/{val}/Emission_data' elif (key == 'datm_ver'): - self.s3dict['cfsr'] = f'datm/{val}/cfsr' - self.s3dict['gefs'] = f'datm/{val}/gefs' - self.s3dict['gfs'] = f'datm/{val}/gfs' - self.s3dict['mom6'] = f'datm/{val}/mom6' + self.s3dict['cfsr']=f'datm/{val}/cfsr' + self.s3dict['gefs']=f'datm/{val}/gefs' + self.s3dict['gfs']=f'datm/{val}/gfs' + self.s3dict['mom6']=f'datm/{val}/mom6' elif (key == 'glwu_ver'): - self.s3dict['glwu'] = f'glwu/{val}' + self.s3dict['glwu']=f'glwu/{val}' elif (key == 'gsi_ver'): - self.s3dict['gsi'] = f'gsi/{val}' + self.s3dict['gsi']=f'gsi/{val}' elif (key == 'lut_ver'): - self.s3dict['lut'] = f'lut/{val}' + self.s3dict['lut']=f'lut/{val}' elif (key == 'mom6_ver'): - self.s3dict['mom6post'] = f'mom6/{val}/post' + self.s3dict['mom6post']=f'mom6/{val}/post' elif (key == 'reg2grb2_ver'): - self.s3dict['reg2grb2'] = f'reg2grb2/{val}' + self.s3dict['reg2grb2']=f'reg2grb2/{val}' elif (key == 'sfc_climb_ver' ): - self.s3dict['sfc_climo'] = f'sfc_climo/{val}' + self.s3dict['sfc_climo']=f'sfc_climo/{val}' elif (key == 'verif_ver'): - self.s3dict['verif'] = f'verif/{val}' + self.s3dict['verif']=f'verif/{val}' elif (key == 'wave_ver'): - self.s3dict['wave'] = f'wave/{val}' + self.s3dict['wave']=f'wave/{val}' # ---------------------------------------------------------------------------------------------------------------- def add_grid_data(self): for key in self.fix_ver_dict.keys(): - val = self.fix_ver_dict[key] + val=self.fix_ver_dict[key] if (key == 'orog_ver'): self.add_atmgrid2s3dict('orog', key, val) elif (key == 'ugwd_ver'): @@ -102,23 +103,23 @@ def add_grid_data(self): def add_atmgrid2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: - newkey = f'{key}_{atmgrid}' - self.s3dict[newkey] = f'{varname}/{val}/{atmgrid}' + newkey=f'{key}_{atmgrid}' + self.s3dict[newkey]=f'{varname}/{val}/{atmgrid}' # ---------------------------------------------------------------------------------------------------------------- def add_ocngrid2s3dict(self, varname, key, val): for ocngrid in self.ocngridarray: - newkey = f'{key}_{atmgrid}' - self.s3dict[newkey] = f'{varname}/{val}/{ocngrid}' + newkey=f'{key}_{atmgrid}' + self.s3dict[newkey]=f'{varname}/{val}/{ocngrid}' # ---------------------------------------------------------------------------------------------------------------- def add_cpl2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: for ocngrid in self.ocngridarray: - newkey = f'{key}_a{atmgrid}o{ocngrid}' - self.s3dict[newkey] = f'{varname}/{val}/a{atmgrid}o{ocngrid}' + newkey=f'{key}_a{atmgrid}o{ocngrid}' + self.s3dict[newkey]=f'{varname}/{val}/a{atmgrid}o{ocngrid}' # ---------------------------------------------------------------------------------------------------------------- def printinfo(self): @@ -129,7 +130,7 @@ def printinfo(self): print(f'From: {self.aws_fix_bucket}') print(f'To: {self.targetdir}') for key in self.s3dict.keys(): - val = self.s3dict[key] + val=self.s3dict[key] print(f'{key}: {val}') # ---------------------------------------------------------------------------------------------------------------- @@ -138,7 +139,7 @@ def fetchdata(self): if (self.verbose): print('Create local fix dir: {self.targetdir}') - path = Path(self.targetdir) + path=Path(self.targetdir) path.mkdir(parents=True, exist_ok=True) self.fetch_ugwp_limb_tau() @@ -149,47 +150,47 @@ def fetchdata(self): # ---------------------------------------------------------------------------------------------------------------- def fetch_dir(self, dir): - remotedir = f'{self.aws_fix_bucket}/{dir}' - localdir = f'{self.targetdir}/{dir}' - cmd = f'{self.aws_sync} {remotedir} {localdir}' + remotedir=f'{self.aws_fix_bucket}/{dir}' + localdir=f'{self.targetdir}/{dir}' + cmd=f'{self.aws_sync} {remotedir} {localdir}' self.download_dir(cmd, localdir) # ---------------------------------------------------------------------------------------------------------------- def download_dir(self, cmd, localdir): - # returned_value = os.system(cmd) # returns the exit code in unix + # returned_value=os.system(cmd) # returns the exit code in unix # print('returned value:', returned_value) if (os.path.isdir(localdir)): print(f'{localdir} already exist. skip' else: - parentdir, dirname = os.path.split(localdir) + parentdir, dirname=os.path.split(localdir) if (self.verbose): print(f'Create local {parentdir} dir:') - path = Path(parentdir) + path=Path(parentdir) path.mkdir(parents=True, exist_ok=True) if (self.verbose): print(cmd) print(f'Downloading {localdir}') - returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix + returned_value=subprocess.call(cmd, shell=True) # returns the exit code in unix if (self.verbose): print('returned value:', returned_value) # ---------------------------------------------------------------------------------------------------------------- def fetch_ugwp_limb_tau(self): - ugwp_limb_tau_remotepath = f'{self.aws_fix_bucket}/ugwd/{self.fix_ver_dict['ugwd_ver']}/ugwp_limb_tau.nc' - ugwp_limb_tau_localdir = f'{self.targetdir}/ugwd/{self.fix_ver_dict['ugwd_ver']}' - filename = f'{ugwp_limb_tau_localdir}/ugwp_limb_tau.nc' - path = Path(ugwp_limb_tau_localdir) + ugwp_limb_tau_remotepath=f'{self.aws_fix_bucket}/ugwd/{self.fix_ver_dict['ugwd_ver']}/ugwp_limb_tau.nc' + ugwp_limb_tau_localdir=f'{self.targetdir}/ugwd/{self.fix_ver_dict['ugwd_ver']}' + filename=f'{ugwp_limb_tau_localdir}/ugwp_limb_tau.nc' + path=Path(ugwp_limb_tau_localdir) path.mkdir(parents=True, exist_ok=True) - cmd = f'{self.aws_cp} {ugwp_limb_tau_remotepath} {filename}' + cmd=f'{self.aws_cp} {ugwp_limb_tau_remotepath} {filename}' self.download_file(cmd, filename) # ---------------------------------------------------------------------------------------------------------------- def download_file(self, cmd, filename): - # returned_value = os.system(cmd) # returns the exit code in unix + # returned_value=os.system(cmd) # returns the exit code in unix # print('returned value:', returned_value) if (os.path.isfile(filename)): @@ -198,29 +199,29 @@ def download_file(self, cmd, filename): if (self.verbose): print(cmd) print(f'Downloading {filename}') - returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix + returned_value= subprocess.call(cmd, shell=True) # returns the exit code in unix if (self.verbose): print('returned value:', returned_value) - # ---------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------------------- def set_fix_ver_from_gwhome(self, gwhome, verdict): - fix_ver_file = f'{gwhome}/versions/fix.ver' - self.fix_ver_dict = verdict + fix_ver_file=f'{gwhome}/versions/fix.ver' + self.fix_ver_dict=verdict if (os.path.isfile(fix_ver_file)): with open(fix_ver_file, "r") as file: for line in file.readlines(): if (line.find('export ') >= 0): - headstr, _, value = line.strip().partition('=') - exphead, _, key = headstr.partition(' ') - self.fix_ver_dict[key] = value + headstr, _, value=line.strip().partition('=') + exphead, _, key=headstr.partition(' ') + self.fix_ver_dict[key]=value else: print(f'fix_ver_file: {ix_ver_file}s does not exist.') - # ---------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------------------------------------------- def set_default_fix_ver(self, verdict): - self.fix_ver_dict = verdict + self.fix_ver_dict=verdict # ---------------------------------------------------------------------------------------------------------------- def print_usage(verdict): @@ -241,78 +242,78 @@ def print_usage(verdict): if __name__ == '__main__': - atmgridlist = ['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] - ocngridlist = ['500', '100', '050', '025'] + atmgridlist=['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] + ocngridlist=['500', '100', '050', '025'] - verbose = 0 - atmgrid = 'C48' - ocngrid = '500' - localdir = '/contrib/global-workflow-shared-data' + verbose=0 + atmgrid='C48' + ocngrid='500' + localdir='/contrib/global-workflow-shared-data' # default fix-version - verdict = {} - verdict['aer_ver'] = '20220805' - verdict['am_ver'] = '20220805' - verdict['chem_ver'] = '20220805' - verdict['cice_ver'] = '20240416' - verdict['cpl_ver'] = '20230526' - verdict['datm_ver'] = '20220805' - verdict['glwu_ver'] = '20220805' - verdict['gsi_ver'] = '20240208' - verdict['lut_ver'] = '20220805' - verdict['mom6_ver'] = '20240416' - verdict['orog_ver'] = '20231027' - verdict['reg2grb2_ver'] = '20220805' - verdict['sfc_climo_ver'] = '20220805' - verdict['ugwd_ver'] = '20240624' - verdict['verif_ver'] = '20220805' - verdict['wave_ver'] = '20240105' - - gwhome = None - - opts, args = getopt.getopt(sys.argv[1:], '', ['help', 'atmgrid=', 'ocngrid=', - 'verbose=', 'localdir=', - 'gwhome=', - 'aer_ver=', - 'am_ver=', - 'chem_ver=', - 'cice_ver=', - 'cpl_ver=', - 'datm_ver=', - 'glwu_ver=', - 'gsi_ver=', - 'lut_ver=', - 'mom6_ver=', - 'orog_ver=', - 'reg2grb2_ver=', - 'sfc_climo_ver=', - 'ugwd_ver=', - 'verif_ver=', - 'wave_ver=']) + verdict={} + verdict['aer_ver']='20220805' + verdict['am_ver']='20220805' + verdict['chem_ver']='20220805' + verdict['cice_ver']='20240416' + verdict['cpl_ver']='20230526' + verdict['datm_ver']='20220805' + verdict['glwu_ver']='20220805' + verdict['gsi_ver']='20240208' + verdict['lut_ver']='20220805' + verdict['mom6_ver']='20240416' + verdict['orog_ver']='20231027' + verdict['reg2grb2_ver']='20220805' + verdict['sfc_climo_ver']='20220805' + verdict['ugwd_ver']='20240624' + verdict['verif_ver']='20220805' + verdict['wave_ver']='20240105' + + gwhome=None + + opts, args=getopt.getopt(sys.argv[1:], '', ['help', 'atmgrid=', 'ocngrid=', + 'verbose=', 'localdir=', + 'gwhome=', + 'aer_ver=', + 'am_ver=', + 'chem_ver=', + 'cice_ver=', + 'cpl_ver=', + 'datm_ver=', + 'glwu_ver=', + 'gsi_ver=', + 'lut_ver=', + 'mom6_ver=', + 'orog_ver=', + 'reg2grb2_ver=', + 'sfc_climo_ver=', + 'ugwd_ver=', + 'verif_ver=', + 'wave_ver=']) for o, a in opts: # print(f'o: {o}, a: {a}') if o in ['--help']: print_usage(verdict) sys.exit(0) elif o in ['--verbose']: - verbose = int(a) + verbose=int(a) elif o in ['--atmgrid']: - atmgrid = a + atmgrid=a elif o in ['--ocngrid']: - ocngrid = a + ocngrid=a elif o in ['--localdir']: - localdir = a + localdir=a elif o in ['--gwhome']: - gwhome = a + gwhome=a else: - _, vername = o.split('--') + _, vername=o.split('--') print(f'vername: <{vername}>') - verdict[vername] = a + verdict[vername]=a if (atmgrid.find(',') > 0): - atmgridarray = atmgrid.split(',') + atmgridarray=atmgrid.split(',') else: - atmgridarray = [atmgrid] + atmgridarray=[atmgrid] for grid in atmgridarray: if (grid not in atmgridlist): @@ -322,9 +323,9 @@ def print_usage(verdict): sys.exit(-1) if (ocngrid.find(',') > 0): - ocngridarray = ocngrid.split(',') + ocngridarray=ocngrid.split(',') else: - ocngridarray = [ocngrid] + ocngridarray=[ocngrid] for grid in ocngridarray: if (grid not in ocngridlist): @@ -334,9 +335,9 @@ def print_usage(verdict): sys.exit(-1) # ------------------------------------------------------------------ - ffd = FetchFIXdata(atmgridarray=atmgridarray, - ocngridarray=ocngridarray, - localdir=localdir, verbose=verbose) + ffd=FetchFIXdata(atmgridarray=atmgridarray, + ocngridarray=ocngridarray, + localdir=localdir, verbose=verbose) if (gwhome is None): ffd.set_default_fix_ver(verdict) @@ -346,3 +347,4 @@ def print_usage(verdict): ffd.update_s3dict() ffd.fetchdata() + From 27ca2f17f70b35280a96ffb919d6c819f609eaf4 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 26 Feb 2025 23:10:55 +0000 Subject: [PATCH 08/27] fix pynorm error 6 --- ush/fetch-fix-data.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index 6d6f8ac4fe..34df3b19ba 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -10,13 +10,13 @@ import subprocess from pathlib import Path -# ---------------------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ class FetchFIXdata(): def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): - self.aws_fix_bucket='s3://noaa-nws-global-pds/fix' + self.aws_fix_bucket= 's3://noaa-nws-global-pds/fix' self.aws_cp='aws --no-sign-request s3 cp' self.aws_sync='aws --no-sign-request s3 sync' @@ -40,7 +40,7 @@ def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, ve else: self.targetdir=self.localdir - # ---------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def update_s3dict(self): self.update_s3dick_grid_independent() @@ -49,14 +49,14 @@ def update_s3dict(self): if (self.verbose): self.printinfo() - # ---------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def update_s3dick_grid_independent(self): for key in self.fix_ver_dict.keys(): val=self.fix_ver_dict[key] if (key == 'aer_ver'): self.s3dict['aer']=f'aer/{val}' - elif ( key == 'am_ver'): + elif (key == 'am_ver'): self.s3dict['am']=f'am/{val}' elif (key == 'chem_ver'): self.s3dict['fimdata_chem']=f'chem/{val}/fimdata_chem' @@ -76,14 +76,14 @@ def update_s3dick_grid_independent(self): self.s3dict['mom6post']=f'mom6/{val}/post' elif (key == 'reg2grb2_ver'): self.s3dict['reg2grb2']=f'reg2grb2/{val}' - elif (key == 'sfc_climb_ver' ): + elif (key == 'sfc_climb_ver'): self.s3dict['sfc_climo']=f'sfc_climo/{val}' elif (key == 'verif_ver'): self.s3dict['verif']=f'verif/{val}' elif (key == 'wave_ver'): self.s3dict['wave']=f'wave/{val}' - # ---------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def add_grid_data(self): for key in self.fix_ver_dict.keys(): @@ -99,21 +99,21 @@ def add_grid_data(self): elif (key == 'cpl_ver'): self.add_cpl2s3dict('cpl', key, val) - # ---------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def add_atmgrid2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: newkey=f'{key}_{atmgrid}' self.s3dict[newkey]=f'{varname}/{val}/{atmgrid}' - # ---------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def add_ocngrid2s3dict(self, varname, key, val): for ocngrid in self.ocngridarray: newkey=f'{key}_{atmgrid}' self.s3dict[newkey]=f'{varname}/{val}/{ocngrid}' - # ---------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def add_cpl2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: @@ -121,7 +121,7 @@ def add_cpl2s3dict(self, varname, key, val): newkey=f'{key}_a{atmgrid}o{ocngrid}' self.s3dict[newkey]=f'{varname}/{val}/a{atmgrid}o{ocngrid}' - # ---------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def printinfo(self): print(f'Preparing to fetch') @@ -133,7 +133,7 @@ def printinfo(self): val=self.s3dict[key] print(f'{key}: {val}') - # ---------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def fetchdata(self): if (self.verbose): @@ -147,7 +147,7 @@ def fetchdata(self): for key in self.s3dict.keys(): self.fetch_dir(self.s3dict[key]) - # ---------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def fetch_dir(self, dir): remotedir=f'{self.aws_fix_bucket}/{dir}' @@ -155,7 +155,7 @@ def fetch_dir(self, dir): cmd=f'{self.aws_sync} {remotedir} {localdir}' self.download_dir(cmd, localdir) - # ---------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def download_dir(self, cmd, localdir): # returned_value=os.system(cmd) # returns the exit code in unix @@ -176,7 +176,7 @@ def download_dir(self, cmd, localdir): if (self.verbose): print('returned value:', returned_value) - # ---------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def fetch_ugwp_limb_tau(self): ugwp_limb_tau_remotepath=f'{self.aws_fix_bucket}/ugwd/{self.fix_ver_dict['ugwd_ver']}/ugwp_limb_tau.nc' @@ -187,7 +187,7 @@ def fetch_ugwp_limb_tau(self): cmd=f'{self.aws_cp} {ugwp_limb_tau_remotepath} {filename}' self.download_file(cmd, filename) - # ---------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------- def download_file(self, cmd, filename): # returned_value=os.system(cmd) # returns the exit code in unix @@ -199,11 +199,11 @@ def download_file(self, cmd, filename): if (self.verbose): print(cmd) print(f'Downloading {filename}') - returned_value= subprocess.call(cmd, shell=True) # returns the exit code in unix + returned_value=subprocess.call(cmd, shell=True) # returns the exit code in unix if (self.verbose): print('returned value:', returned_value) - # -------------------------------------------------------------------------------------------------------------- + # -------------------------------------------------------------------------- def set_fix_ver_from_gwhome(self, gwhome, verdict): fix_ver_file=f'{gwhome}/versions/fix.ver' @@ -218,12 +218,12 @@ def set_fix_ver_from_gwhome(self, gwhome, verdict): else: print(f'fix_ver_file: {ix_ver_file}s does not exist.') - # -------------------------------------------------------------------------------------------------------------- + # ------------------------------------------------------------------------ def set_default_fix_ver(self, verdict): self.fix_ver_dict=verdict -# ---------------------------------------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- def print_usage(verdict): @@ -238,7 +238,7 @@ def print_usage(verdict): for key in verdict.keys(): print(f'\t--{key}=yyyymmdd default: {verdict[key]}') -# ---------------------------------------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ if __name__ == '__main__': @@ -347,4 +347,3 @@ def print_usage(verdict): ffd.update_s3dict() ffd.fetchdata() - From f39bfd9f4290a08a7b69955e7f61ff2681f3f39e Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 27 Feb 2025 00:18:28 +0000 Subject: [PATCH 09/27] fix pynorm error 7 --- ush/fetch-fix-data.py | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index 34df3b19ba..7bd90026aa 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -13,17 +13,16 @@ # ------------------------------------------------------------------------------ class FetchFIXdata(): - def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): - self.aws_fix_bucket= 's3://noaa-nws-global-pds/fix' - self.aws_cp='aws --no-sign-request s3 cp' - self.aws_sync='aws --no-sign-request s3 sync' + self.aws_fix_bucket= f's3://noaa-nws-global-pds/fix' + self.aws_cp = f'aws --no-sign-request s3 cp' + self.aws_sync = f'aws --no-sign-request s3 sync' - self.atmgridarray=atmgridarray - self.ocngridarray=ocngridarray - self.localdir=localdir - self.verbose=verbose + self.atmgridarray = atmgridarray + self.ocngridarray = ocngridarray + self.localdir = localdir + self.verbose = verbose # if (os.path.isdir(localdir)): # print('Prepare to download FIX data for %s and %s to %s' %(atmgrid, ocngrid, localdir)) @@ -31,14 +30,14 @@ def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, ve # print(f'local dir: <{localdir}> does not exist. Stop') # sys.exit(-1) - self.verdict={} - self.s3dict={} - self.s3dict['raworog']='raw/orog' + self.verdict = {} + self.s3dict = {} + self.s3dict['raworog'] = f'raw/orog' if (self.localdir.find('fix') < 0): - self.targetdir=f'{self.localdir}/fix.subset' + self.targetdir = f'{self.localdir}/fix.subset' else: - self.targetdir=self.localdir + self.targetdir = self.localdir # -------------------------------------------------------------------------- def update_s3dict(self): @@ -118,8 +117,8 @@ def add_cpl2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: for ocngrid in self.ocngridarray: - newkey=f'{key}_a{atmgrid}o{ocngrid}' - self.s3dict[newkey]=f'{varname}/{val}/a{atmgrid}o{ocngrid}' + newkey = f'{key}_a{atmgrid}o{ocngrid}' + self.s3dict[newkey] = f'{varname}/{val}/a{atmgrid}o{ocngrid}' # ------------------------------------------------------------------------- def printinfo(self): @@ -137,13 +136,13 @@ def printinfo(self): def fetchdata(self): if (self.verbose): - print('Create local fix dir: {self.targetdir}') + print(f'Create local fix dir: {self.targetdir}') path=Path(self.targetdir) path.mkdir(parents=True, exist_ok=True) self.fetch_ugwp_limb_tau() - + for key in self.s3dict.keys(): self.fetch_dir(self.s3dict[key]) @@ -162,7 +161,7 @@ def download_dir(self, cmd, localdir): # print('returned value:', returned_value) if (os.path.isdir(localdir)): - print(f'{localdir} already exist. skip' + print(f'{localdir} already exist. skip') else: parentdir, dirname=os.path.split(localdir) if (self.verbose): @@ -190,8 +189,8 @@ def fetch_ugwp_limb_tau(self): # ------------------------------------------------------------------------- def download_file(self, cmd, filename): - # returned_value=os.system(cmd) # returns the exit code in unix - # print('returned value:', returned_value) + # returned_value=os.system(cmd) # returns the exit code in unix + # print('returned value:', returned_value) if (os.path.isfile(filename)): print(f'{filename} already exist. skip') @@ -226,7 +225,6 @@ def set_default_fix_ver(self, verdict): # ----------------------------------------------------------------------------- def print_usage(verdict): - print('Usage: python fetch-fix-data.py \\') print(' --atmgrid=AtmospericGrid (for multiple grids, separate with ",") \\') print(' --ocngrid=OceanGrid (for multiple grids, separate with ",") \\') @@ -241,7 +239,6 @@ def print_usage(verdict): # ------------------------------------------------------------------------------ if __name__ == '__main__': - atmgridlist=['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] ocngridlist=['500', '100', '050', '025'] @@ -345,5 +342,4 @@ def print_usage(verdict): ffd.set_fix_ver_from_gwhome(gwhome, verdict) ffd.update_s3dict() - ffd.fetchdata() From 8f7fc23bfc7b2cec2f4b0cab7aa87d9b1417a2d7 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 27 Feb 2025 00:26:42 +0000 Subject: [PATCH 10/27] fix pynorm error 8 --- ush/fetch-fix-data.py | 211 +++++++++++++++++++++--------------------- 1 file changed, 107 insertions(+), 104 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index 7bd90026aa..97a649443b 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -13,9 +13,10 @@ # ------------------------------------------------------------------------------ class FetchFIXdata(): + def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): - self.aws_fix_bucket= f's3://noaa-nws-global-pds/fix' + self.aws_fix_bucket = f's3://noaa-nws-global-pds/fix' self.aws_cp = f'aws --no-sign-request s3 cp' self.aws_sync = f'aws --no-sign-request s3 sync' @@ -52,41 +53,41 @@ def update_s3dict(self): def update_s3dick_grid_independent(self): for key in self.fix_ver_dict.keys(): - val=self.fix_ver_dict[key] + val = self.fix_ver_dict[key] if (key == 'aer_ver'): - self.s3dict['aer']=f'aer/{val}' + self.s3dict['aer'] = f'aer/{val}' elif (key == 'am_ver'): - self.s3dict['am']=f'am/{val}' + self.s3dict['am'] = f'am/{val}' elif (key == 'chem_ver'): - self.s3dict['fimdata_chem']=f'chem/{val}/fimdata_chem' - self.s3dict['Emission_data']=f'chem/{val}/Emission_data' + self.s3dict['fimdata_chem'] = f'chem/{val}/fimdata_chem' + self.s3dict['Emission_data'] = f'chem/{val}/Emission_data' elif (key == 'datm_ver'): - self.s3dict['cfsr']=f'datm/{val}/cfsr' - self.s3dict['gefs']=f'datm/{val}/gefs' - self.s3dict['gfs']=f'datm/{val}/gfs' - self.s3dict['mom6']=f'datm/{val}/mom6' + self.s3dict['cfsr'] = f'datm/{val}/cfsr' + self.s3dict['gefs'] = f'datm/{val}/gefs' + self.s3dict['gfs'] = f'datm/{val}/gfs' + self.s3dict['mom6'] = f'datm/{val}/mom6' elif (key == 'glwu_ver'): - self.s3dict['glwu']=f'glwu/{val}' + self.s3dict['glwu'] = f'glwu/{val}' elif (key == 'gsi_ver'): - self.s3dict['gsi']=f'gsi/{val}' + self.s3dict['gsi'] = f'gsi/{val}' elif (key == 'lut_ver'): - self.s3dict['lut']=f'lut/{val}' + self.s3dict['lut'] = f'lut/{val}' elif (key == 'mom6_ver'): - self.s3dict['mom6post']=f'mom6/{val}/post' + self.s3dict['mom6post'] = f'mom6/{val}/post' elif (key == 'reg2grb2_ver'): - self.s3dict['reg2grb2']=f'reg2grb2/{val}' + self.s3dict['reg2grb2'] = f'reg2grb2/{val}' elif (key == 'sfc_climb_ver'): - self.s3dict['sfc_climo']=f'sfc_climo/{val}' + self.s3dict['sfc_climo'] = f'sfc_climo/{val}' elif (key == 'verif_ver'): - self.s3dict['verif']=f'verif/{val}' + self.s3dict['verif'] = f'verif/{val}' elif (key == 'wave_ver'): - self.s3dict['wave']=f'wave/{val}' + self.s3dict['wave'] = f'wave/{val}' # -------------------------------------------------------------------------- def add_grid_data(self): for key in self.fix_ver_dict.keys(): - val=self.fix_ver_dict[key] + val = self.fix_ver_dict[key] if (key == 'orog_ver'): self.add_atmgrid2s3dict('orog', key, val) elif (key == 'ugwd_ver'): @@ -102,15 +103,15 @@ def add_grid_data(self): def add_atmgrid2s3dict(self, varname, key, val): for atmgrid in self.atmgridarray: - newkey=f'{key}_{atmgrid}' - self.s3dict[newkey]=f'{varname}/{val}/{atmgrid}' + newkey = f'{key}_{atmgrid}' + self.s3dict[newkey] = f'{varname}/{val}/{atmgrid}' # ------------------------------------------------------------------------- def add_ocngrid2s3dict(self, varname, key, val): for ocngrid in self.ocngridarray: - newkey=f'{key}_{atmgrid}' - self.s3dict[newkey]=f'{varname}/{val}/{ocngrid}' + newkey = f'{key}_{atmgrid}' + self.s3dict[newkey] = f'{varname}/{val}/{ocngrid}' # ------------------------------------------------------------------------- def add_cpl2s3dict(self, varname, key, val): @@ -129,7 +130,7 @@ def printinfo(self): print(f'From: {self.aws_fix_bucket}') print(f'To: {self.targetdir}') for key in self.s3dict.keys(): - val=self.s3dict[key] + val = self.s3dict[key] print(f'{key}: {val}') # ------------------------------------------------------------------------- @@ -138,7 +139,7 @@ def fetchdata(self): if (self.verbose): print(f'Create local fix dir: {self.targetdir}') - path=Path(self.targetdir) + path = Path(self.targetdir) path.mkdir(parents=True, exist_ok=True) self.fetch_ugwp_limb_tau() @@ -149,47 +150,47 @@ def fetchdata(self): # ------------------------------------------------------------------------- def fetch_dir(self, dir): - remotedir=f'{self.aws_fix_bucket}/{dir}' - localdir=f'{self.targetdir}/{dir}' - cmd=f'{self.aws_sync} {remotedir} {localdir}' + remotedir = f'{self.aws_fix_bucket}/{dir}' + localdir = f'{self.targetdir}/{dir}' + cmd = f'{self.aws_sync} {remotedir} {localdir}' self.download_dir(cmd, localdir) # -------------------------------------------------------------------------- def download_dir(self, cmd, localdir): - # returned_value=os.system(cmd) # returns the exit code in unix + # returned_value = os.system(cmd) # returns the exit code in unix # print('returned value:', returned_value) if (os.path.isdir(localdir)): print(f'{localdir} already exist. skip') else: - parentdir, dirname=os.path.split(localdir) + parentdir, dirname = os.path.split(localdir) if (self.verbose): print(f'Create local {parentdir} dir:') - path=Path(parentdir) + path = Path(parentdir) path.mkdir(parents=True, exist_ok=True) if (self.verbose): print(cmd) print(f'Downloading {localdir}') - returned_value=subprocess.call(cmd, shell=True) # returns the exit code in unix + returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix if (self.verbose): print('returned value:', returned_value) # -------------------------------------------------------------------------- def fetch_ugwp_limb_tau(self): - ugwp_limb_tau_remotepath=f'{self.aws_fix_bucket}/ugwd/{self.fix_ver_dict['ugwd_ver']}/ugwp_limb_tau.nc' - ugwp_limb_tau_localdir=f'{self.targetdir}/ugwd/{self.fix_ver_dict['ugwd_ver']}' - filename=f'{ugwp_limb_tau_localdir}/ugwp_limb_tau.nc' - path=Path(ugwp_limb_tau_localdir) + ugwp_limb_tau_remotepath = f'{self.aws_fix_bucket}/ugwd/{self.fix_ver_dict['ugwd_ver']}/ugwp_limb_tau.nc' + ugwp_limb_tau_localdir = f'{self.targetdir}/ugwd/{self.fix_ver_dict['ugwd_ver']}' + filename = f'{ugwp_limb_tau_localdir}/ugwp_limb_tau.nc' + path = Path(ugwp_limb_tau_localdir) path.mkdir(parents=True, exist_ok=True) - cmd=f'{self.aws_cp} {ugwp_limb_tau_remotepath} {filename}' + cmd = f'{self.aws_cp} {ugwp_limb_tau_remotepath} {filename}' self.download_file(cmd, filename) # ------------------------------------------------------------------------- def download_file(self, cmd, filename): - # returned_value=os.system(cmd) # returns the exit code in unix + # returned_value = os.system(cmd) # returns the exit code in unix # print('returned value:', returned_value) if (os.path.isfile(filename)): @@ -198,33 +199,34 @@ def download_file(self, cmd, filename): if (self.verbose): print(cmd) print(f'Downloading {filename}') - returned_value=subprocess.call(cmd, shell=True) # returns the exit code in unix + returned_value = subprocess.call(cmd, shell=True) # returns the exit code in unix if (self.verbose): print('returned value:', returned_value) # -------------------------------------------------------------------------- def set_fix_ver_from_gwhome(self, gwhome, verdict): - fix_ver_file=f'{gwhome}/versions/fix.ver' - self.fix_ver_dict=verdict + fix_ver_file = f'{gwhome}/versions/fix.ver' + self.fix_ver_dict = verdict if (os.path.isfile(fix_ver_file)): with open(fix_ver_file, "r") as file: for line in file.readlines(): if (line.find('export ') >= 0): - headstr, _, value=line.strip().partition('=') - exphead, _, key=headstr.partition(' ') - self.fix_ver_dict[key]=value + headstr, _, value = line.strip().partition('=') + exphead, _, key = headstr.partition(' ') + self.fix_ver_dict[key] = value else: print(f'fix_ver_file: {ix_ver_file}s does not exist.') # ------------------------------------------------------------------------ def set_default_fix_ver(self, verdict): - self.fix_ver_dict=verdict + self.fix_ver_dict = verdict # ----------------------------------------------------------------------------- def print_usage(verdict): + print('Usage: python fetch-fix-data.py \\') print(' --atmgrid=AtmospericGrid (for multiple grids, separate with ",") \\') print(' --ocngrid=OceanGrid (for multiple grids, separate with ",") \\') @@ -239,78 +241,79 @@ def print_usage(verdict): # ------------------------------------------------------------------------------ if __name__ == '__main__': - atmgridlist=['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] - ocngridlist=['500', '100', '050', '025'] - verbose=0 - atmgrid='C48' - ocngrid='500' - localdir='/contrib/global-workflow-shared-data' + atmgridlist = ['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] + ocngridlist = ['500', '100', '050', '025'] + + verbose = 0 + atmgrid = f'C48' + ocngrid = f'500' + localdir = f'/contrib/global-workflow-shared-data' # default fix-version - verdict={} - verdict['aer_ver']='20220805' - verdict['am_ver']='20220805' - verdict['chem_ver']='20220805' - verdict['cice_ver']='20240416' - verdict['cpl_ver']='20230526' - verdict['datm_ver']='20220805' - verdict['glwu_ver']='20220805' - verdict['gsi_ver']='20240208' - verdict['lut_ver']='20220805' - verdict['mom6_ver']='20240416' - verdict['orog_ver']='20231027' - verdict['reg2grb2_ver']='20220805' - verdict['sfc_climo_ver']='20220805' - verdict['ugwd_ver']='20240624' - verdict['verif_ver']='20220805' - verdict['wave_ver']='20240105' - - gwhome=None - - opts, args=getopt.getopt(sys.argv[1:], '', ['help', 'atmgrid=', 'ocngrid=', - 'verbose=', 'localdir=', - 'gwhome=', - 'aer_ver=', - 'am_ver=', - 'chem_ver=', - 'cice_ver=', - 'cpl_ver=', - 'datm_ver=', - 'glwu_ver=', - 'gsi_ver=', - 'lut_ver=', - 'mom6_ver=', - 'orog_ver=', - 'reg2grb2_ver=', - 'sfc_climo_ver=', - 'ugwd_ver=', - 'verif_ver=', - 'wave_ver=']) + verdict = {} + verdict['aer_ver'] = f'20220805' + verdict['am_ver'] = f'20220805' + verdict['chem_ver'] = f'20220805' + verdict['cice_ver'] = f'20240416' + verdict['cpl_ver'] = f'20230526' + verdict['datm_ver'] = f'20220805' + verdict['glwu_ver'] = f'20220805' + verdict['gsi_ver'] = f'20240208' + verdict['lut_ver'] = f'20220805' + verdict['mom6_ver'] = f'20240416' + verdict['orog_ver'] = f'20231027' + verdict['reg2grb2_ver'] = f'20220805' + verdict['sfc_climo_ver'] = f'20220805' + verdict['ugwd_ver'] = f'20240624' + verdict['verif_ver'] = f'20220805' + verdict['wave_ver'] = f'20240105' + + gwhome = None + + opts, args = getopt.getopt(sys.argv[1:], '', ['help', 'atmgrid=', 'ocngrid=', + 'verbose=', 'localdir=', + 'gwhome=', + 'aer_ver=', + 'am_ver=', + 'chem_ver=', + 'cice_ver=', + 'cpl_ver=', + 'datm_ver=', + 'glwu_ver=', + 'gsi_ver=', + 'lut_ver=', + 'mom6_ver=', + 'orog_ver=', + 'reg2grb2_ver=', + 'sfc_climo_ver=', + 'ugwd_ver=', + 'verif_ver=', + 'wave_ver=']) for o, a in opts: # print(f'o: {o}, a: {a}') if o in ['--help']: print_usage(verdict) sys.exit(0) elif o in ['--verbose']: - verbose=int(a) + verbose = int(a) elif o in ['--atmgrid']: - atmgrid=a + atmgrid = a elif o in ['--ocngrid']: - ocngrid=a + ocngrid = a elif o in ['--localdir']: - localdir=a + localdir = a elif o in ['--gwhome']: - gwhome=a + gwhome = a else: - _, vername=o.split('--') + _, vername = o.split('--') print(f'vername: <{vername}>') - verdict[vername]=a + verdict[vername] = a if (atmgrid.find(',') > 0): - atmgridarray=atmgrid.split(',') + atmgridarray = atmgrid.split(',') else: - atmgridarray=[atmgrid] + atmgridarray = [atmgrid] for grid in atmgridarray: if (grid not in atmgridlist): @@ -320,9 +323,9 @@ def print_usage(verdict): sys.exit(-1) if (ocngrid.find(',') > 0): - ocngridarray=ocngrid.split(',') + ocngridarray = ocngrid.split(',') else: - ocngridarray=[ocngrid] + ocngridarray = [ocngrid] for grid in ocngridarray: if (grid not in ocngridlist): @@ -332,9 +335,9 @@ def print_usage(verdict): sys.exit(-1) # ------------------------------------------------------------------ - ffd=FetchFIXdata(atmgridarray=atmgridarray, - ocngridarray=ocngridarray, - localdir=localdir, verbose=verbose) + ffd = FetchFIXdata(atmgridarray=atmgridarray, + ocngridarray=ocngridarray, + localdir=localdir, verbose=verbose) if (gwhome is None): ffd.set_default_fix_ver(verdict) From c248e128c38079e09c9a54fcf5e69aad1036ec3a Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 27 Feb 2025 00:30:10 +0000 Subject: [PATCH 11/27] fix pynorm error 9 --- ush/fetch-fix-data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index 97a649443b..f75c80c9b0 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -11,8 +11,8 @@ from pathlib import Path # ------------------------------------------------------------------------------ -class FetchFIXdata(): +class FetchFIXdata(): def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): @@ -224,8 +224,8 @@ def set_default_fix_ver(self, verdict): self.fix_ver_dict = verdict # ----------------------------------------------------------------------------- -def print_usage(verdict): +def print_usage(verdict): print('Usage: python fetch-fix-data.py \\') print(' --atmgrid=AtmospericGrid (for multiple grids, separate with ",") \\') @@ -239,8 +239,8 @@ def print_usage(verdict): print(f'\t--{key}=yyyymmdd default: {verdict[key]}') # ------------------------------------------------------------------------------ -if __name__ == '__main__': +if __name__ == '__main__': atmgridlist = ['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] ocngridlist = ['500', '100', '050', '025'] From 85120f3398662440b804996c63695567247bbea9 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 27 Feb 2025 00:37:53 +0000 Subject: [PATCH 12/27] fix pynorm error 10 --- ush/fetch-fix-data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/fetch-fix-data.py b/ush/fetch-fix-data.py index f75c80c9b0..e920723455 100644 --- a/ush/fetch-fix-data.py +++ b/ush/fetch-fix-data.py @@ -12,6 +12,7 @@ # ------------------------------------------------------------------------------ + class FetchFIXdata(): def __init__(self, atmgridarray=['C48'], ocngridarray=['500'], localdir=None, verbose=0): @@ -225,6 +226,7 @@ def set_default_fix_ver(self, verdict): # ----------------------------------------------------------------------------- + def print_usage(verdict): print('Usage: python fetch-fix-data.py \\') @@ -240,6 +242,7 @@ def print_usage(verdict): # ------------------------------------------------------------------------------ + if __name__ == '__main__': atmgridlist = ['C48', 'C96', 'C192', 'C384', 'C768', 'C1152'] From 2a7f1c9c103807fe8eef56741536286812bf15fd Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 27 Feb 2025 21:17:36 +0000 Subject: [PATCH 13/27] fix a syntax error --- ci/Jenkinsfile4AWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index c23ace3a96..731215a566 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -6,7 +6,7 @@ def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: /lustre/jenkins/global-workflow/CI] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins/global-workflow/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' From 9f8ee2dd690e24c5d565ff08fb64c09b886cf4b8 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 27 Feb 2025 21:21:02 +0000 Subject: [PATCH 14/27] use just /lutre/jenkins for CI testing work directory --- ci/Jenkinsfile4AWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index 731215a566..5efc18bf89 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -6,7 +6,7 @@ def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins/global-workflow/CI'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' From 0373d6f96dbec1fc4599b199e0121f02393c1552 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 13:15:14 -0700 Subject: [PATCH 15/27] Update --- ci/Jenkinsfile4AWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index 5efc18bf89..bc8521f908 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -5,8 +5,8 @@ def HOMEgfs = 'none' def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins'] +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', awsepicglobalworkflow: 'awsepicglobalworkflow'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', awsepicglobalworkflow: '/lustre/jenkins'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' From ff798459e8ab33f966f4e4b39a7f8238f8b06eff Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 13:30:01 -0700 Subject: [PATCH 16/27] Update. --- ci/Jenkinsfile4AWS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index bc8521f908..cfbb086aae 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -5,9 +5,9 @@ def HOMEgfs = 'none' def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', awsepicglobalworkflow: 'awsepicglobalworkflow'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', awsepicglobalworkflow: '/lustre/jenkins'] -def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins'] +def repo_url = 'git@github.com:NOAA-EPIC/global-workflow-cloud.git' def STATUS = 'Passed' pipeline { From 50bb9073f123c4f32f27a7886ccaa8cd94d7682a Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 13:50:36 -0700 Subject: [PATCH 17/27] Updating label. --- ci/Jenkinsfile4AWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index cfbb086aae..d1c696f4c7 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -5,8 +5,8 @@ def HOMEgfs = 'none' def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins'] +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', awsepicglobalworkflow: 'awsepicglobalworkflow'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', awsepicglobalworkflow: '/lustre/jenkins'] def repo_url = 'git@github.com:NOAA-EPIC/global-workflow-cloud.git' def STATUS = 'Passed' From 02c6fa4b1536a85d6a0f87367767432a035c6110 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 13:58:57 -0700 Subject: [PATCH 18/27] Update --- ci/Jenkinsfile4AWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index d1c696f4c7..2d1d2f3a4c 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -56,7 +56,7 @@ pipeline { run_nodes.init().each { node -> def machine_name = node.split('-')[0].toLowerCase() echo "Spawning job on node: ${node} with machine name: ${machine_name}" - build job: "/global-workflow/EMC-Global-Pipeline/PR-${env.CHANGE_ID}", parameters: [ + build job: "/global-workflow/EPIC-AWS-Global-Workflow-Pipeline-Test/PR-${env.CHANGE_ID}", parameters: [ string(name: 'machine', value: machine_name), string(name: 'Node', value: node) ], wait: false From 08a2ee5a027aec0e3c94cb2ccd5738d060e8a721 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 14:05:46 -0700 Subject: [PATCH 19/27] Update --- ci/Jenkinsfile4AWS | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index 2d1d2f3a4c..50e0631b37 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -42,6 +42,7 @@ pipeline { } else { echo "This is parent job so getting list of nodes matching labels:" for (label in pullRequest.labels) { + print label if (label.matches("CI-(.*?)-Ready")) { def machine_name = label.split('-')[1].toString().toLowerCase() jenkins.model.Jenkins.get().computers.each { c -> From 2c6bd875911e0e062e05c96acad726a8e1ca63ba Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 14:13:41 -0700 Subject: [PATCH 20/27] Update --- ci/Jenkinsfile4AWS | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index 50e0631b37..e64e693fd9 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -45,6 +45,7 @@ pipeline { print label if (label.matches("CI-(.*?)-Ready")) { def machine_name = label.split('-')[1].toString().toLowerCase() + print machine_name jenkins.model.Jenkins.get().computers.each { c -> if (c.node.selfLabel.name == NodeName[machine_name]) { run_nodes.add(c.node.selfLabel.name) From 64769abae1e0611ce0a1fb85fe0d313551d1a2dc Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 14:28:33 -0700 Subject: [PATCH 21/27] Update --- ci/Jenkinsfile4AWS | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index e64e693fd9..c0d883850e 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -57,6 +57,7 @@ pipeline { if (run_nodes.size() > 1) { run_nodes.init().each { node -> def machine_name = node.split('-')[0].toLowerCase() + print machine_name echo "Spawning job on node: ${node} with machine name: ${machine_name}" build job: "/global-workflow/EPIC-AWS-Global-Workflow-Pipeline-Test/PR-${env.CHANGE_ID}", parameters: [ string(name: 'machine', value: machine_name), From cfb41321bfe40e9f6ee03996d50d8fedb1a0f27b Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 14:31:16 -0700 Subject: [PATCH 22/27] Updating. --- ci/Jenkinsfile4AWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index c0d883850e..6b0596e84c 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -49,6 +49,7 @@ pipeline { jenkins.model.Jenkins.get().computers.each { c -> if (c.node.selfLabel.name == NodeName[machine_name]) { run_nodes.add(c.node.selfLabel.name) + print c.node.selfLabel.name } } } @@ -57,7 +58,6 @@ pipeline { if (run_nodes.size() > 1) { run_nodes.init().each { node -> def machine_name = node.split('-')[0].toLowerCase() - print machine_name echo "Spawning job on node: ${node} with machine name: ${machine_name}" build job: "/global-workflow/EPIC-AWS-Global-Workflow-Pipeline-Test/PR-${env.CHANGE_ID}", parameters: [ string(name: 'machine', value: machine_name), From 7feb03f42dba0c14f2ec533c353689877e0274c2 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 14:34:33 -0700 Subject: [PATCH 23/27] Updating. --- ci/Jenkinsfile4AWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index 6b0596e84c..52a247ddb1 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -47,9 +47,10 @@ pipeline { def machine_name = label.split('-')[1].toString().toLowerCase() print machine_name jenkins.model.Jenkins.get().computers.each { c -> + print c.node.selfLabel.name + print NodeName[machine_name] if (c.node.selfLabel.name == NodeName[machine_name]) { run_nodes.add(c.node.selfLabel.name) - print c.node.selfLabel.name } } } From 61b57f1e39df95f975b93f3557434a93d5698c91 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 14:37:41 -0700 Subject: [PATCH 24/27] Update. --- ci/Jenkinsfile4AWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index 52a247ddb1..b5124047c5 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -47,10 +47,10 @@ pipeline { def machine_name = label.split('-')[1].toString().toLowerCase() print machine_name jenkins.model.Jenkins.get().computers.each { c -> - print c.node.selfLabel.name - print NodeName[machine_name] if (c.node.selfLabel.name == NodeName[machine_name]) { run_nodes.add(c.node.selfLabel.name) + print "condition matched" + print run_nodes.size() } } } From 36e00a29c4f7c58f867fc8f8586041872442dc42 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 15:48:57 -0700 Subject: [PATCH 25/27] Updating. --- ci/Jenkinsfile4AWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index b5124047c5..d387cd0ce9 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -5,8 +5,8 @@ def HOMEgfs = 'none' def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', awsepicglobalworkflow: 'awsepicglobalworkflow'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', awsepicglobalworkflow: '/lustre/jenkins'] +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins'] def repo_url = 'git@github.com:NOAA-EPIC/global-workflow-cloud.git' def STATUS = 'Passed' From 4149ec0aa201c3031fd8cf7c7e7899e6166badd1 Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 15:51:19 -0700 Subject: [PATCH 26/27] Reverting changes. --- ci/Jenkinsfile4AWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index d387cd0ce9..b5124047c5 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -5,8 +5,8 @@ def HOMEgfs = 'none' def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', noaacloud: 'awsepicglobalworkflow'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', noaacloud: '/lustre/jenkins'] +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', awsepicglobalworkflow: 'awsepicglobalworkflow'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', awsepicglobalworkflow: '/lustre/jenkins'] def repo_url = 'git@github.com:NOAA-EPIC/global-workflow-cloud.git' def STATUS = 'Passed' From 101741411d778e4b553e78edeed66c43289829bd Mon Sep 17 00:00:00 2001 From: Kris Booker Date: Fri, 28 Feb 2025 15:56:55 -0700 Subject: [PATCH 27/27] Updating. --- ci/Jenkinsfile4AWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/Jenkinsfile4AWS b/ci/Jenkinsfile4AWS index b5124047c5..ac7b38e010 100644 --- a/ci/Jenkinsfile4AWS +++ b/ci/Jenkinsfile4AWS @@ -5,7 +5,7 @@ def HOMEgfs = 'none' def CI_CASES = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persistent for each iteration of the PR. -def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', awsepicglobalworkflow: 'awsepicglobalworkflow'] +def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea', awsepicglobalworkflow: 'Awsepicglobalworkflow'] def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI', awsepicglobalworkflow: '/lustre/jenkins'] def repo_url = 'git@github.com:NOAA-EPIC/global-workflow-cloud.git' def STATUS = 'Passed'