add ioda convert capability to prep_obs

ufs-community · Nov 22, 2024 · 4d99e62 · 4d99e62
1 parent 08494fe
commit 4d99e62
Show file tree

Hide file tree

Showing 23 changed files with 376 additions and 138 deletions.
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,7 @@ sorc/lib/
 sorc/lib64
 parm/conda_loc
 parm/config.yaml
+ush/__pycache__/
 
 __pycache__
 *.swp

diff --git a/README.md b/README.md
@@ -1,12 +1,12 @@
-# UFS Offline Land Data Assimilation System
+# UFS Land Data Assimilation System
 
 The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. It is designed to be the source system for NOAA's operational numerical weather prediction applications while enabling research, development, and contribution opportunities for the broader Weather Enterprise. For more information about the UFS, visit the UFS Portal at https://ufs.epic.noaa.gov/.
 
-The UFS includes [multiple applications](https://ufs.epic.noaa.gov/applications/) that support different forecast durations and spatial domains. This repository hosts the source code for the UFS Land Data Assimilation (DA) System. Land DA is an offline version of the Noah Multi-Physics (Noah-MP) land surface model (LSM) used in the UFS Weather Model (WM). Its data assimilation framework uses the Joint Effort for Data assimilation Integration (JEDI) software stack, which includes the Object-Oriented Prediction System (OOPS) for the data assimilation algorithm, the Interface for Observation Data Access (IODA) for observation formatting and processing, and the Unified Forward Operator (UFO) for comparing model forecasts and observations. 
+The UFS includes [multiple applications](https://ufs.epic.noaa.gov/applications/) that support different forecast durations and spatial domains. This repository hosts the source code for the UFS Land Data Assimilation (DA) System. Land DA applies the Noah Multi-Physics (Noah-MP) land surface model (LSM) of the UFS Weather Model (WM) as the key component. Its data assimilation framework uses the Joint Effort for Data assimilation Integration (JEDI) software stack, which includes the Object-Oriented Prediction System (OOPS) for the data assimilation algorithm, the Interface for Observation Data Access (IODA) for observation formatting and processing, and the Unified Forward Operator (UFO) for comparing model forecasts and observations. 
 
-The offline Noah-MP LSM is a standalone, uncoupled model used to execute land surface simulations. In this traditional uncoupled mode, near-surface atmospheric forcing data is required as input forcing. This LSM simulates soil moisture (both liquid and frozen), soil temperature, skin temperature, snow depth, snow water equivalent (SWE), snow density, canopy water content, and the energy flux and water flux terms of the surface energy balance and surface water balance. Its data assimilation framework applies the Local Ensemble Transform Kalman Filter-Optimal Interpolation (LETKF-OI) algorithm to combine the state-dependent background error derived from an ensemble forecast with the observations and their corresponding uncertainties to produce an analysis ensemble (Hunt et al., 2007).
+The Noah-MP LSM is used to execute land surface simulations. A near-surface atmospheric forcing data is required as input forcing. This LSM simulates soil moisture (both liquid and frozen), soil temperature, skin temperature, snow depth, snow water equivalent (SWE), snow density, canopy water content, and the energy flux and water flux terms of the surface energy balance and surface water balance. Its data assimilation framework applies the Local Ensemble Transform Kalman Filter-Optimal Interpolation (LETKF-OI) algorithm to combine the state-dependent background error derived from an ensemble forecast with the observations and their corresponding uncertainties to produce an analysis ensemble (Hunt et al., 2007).
 
-The Noah-MP LSM has evolved through community efforts to pursue and refine a modern-era LSM suitable for use in the National Centers for Environmental Prediction (NCEP) operational weather and climate prediction models. This collaborative effort continues with participation from entities such as NCAR, NCEP, NASA, and university groups. The development branch of the Land DA System is continually evolving as the system undergoes open development. The latest Land DA release (v1.2.0) represents a snapshot of this continuously evolving system. 
+The Noah-MP LSM has evolved through community efforts to pursue and refine a modern-era LSM suitable for use in the National Centers for Environmental Prediction (NCEP) operational weather and climate prediction models. This collaborative effort continues with participation from entities such as NCAR, NCEP, NASA, and university groups. The development branch of the Land DA System is continually evolving as the system undergoes open development. The latest Land DA release (v2.0.0) represents a snapshot of this continuously evolving system. 
 
 The Land DA System User's Guide associated with the development branch is at: https://land-da-workflow.readthedocs.io/en/develop/, while the guide specific to the Land DA v2.0.0 release can be found at: https://land-da-workflow.readthedocs.io/en/release-public-v2.0.0/. Users may download data for use with the most recent release from the [Land DA data bucket](https://registry.opendata.aws/noaa-ufs-land-da/). The [Land DA Docker Hub](https://hub.docker.com/r/noaaepic/ubuntu22.04-intel21.10-landda) hosts Land DA containers. These containers package the Land DA System together with all its software dependencies for an easier experience building and running Land DA.
 

diff --git a/jobs/JLANDDA_ANALYSIS b/jobs/JLANDDA_ANALYSIS
@@ -80,8 +80,6 @@ mkdir -p ${DATA_SHARE}
 export DATA_HOFX="${DATA_HOFX:-${DATAROOT}/DATA_SHARE/hofx}"
 mkdir -p ${DATA_HOFX}
 
-# Set other dates
-export PTIME=$($NDATE -24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/jobs/JLANDDA_FORECAST b/jobs/JLANDDA_FORECAST
@@ -83,8 +83,6 @@ mkdir -p ${DATA_SHARE}
 export DATA_RESTART="${DATA_RESTART:-${DATAROOT}/DATA_SHARE/RESTART}"
 mkdir -p ${DATA_RESTART}
 
-# Set other dates
-export NTIME=$($NDATE 24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/jobs/JLANDDA_PLOT_STATS b/jobs/JLANDDA_PLOT_STATS
@@ -80,8 +80,6 @@ mkdir -p ${DATA_SHARE}
 export DATA_HOFX="${DATA_HOFX:-${DATAROOT}/DATA_SHARE/hofx}"
 mkdir -p ${DATA_HOFX}
 
-# Set other dates
-export NTIME=$($NDATE 24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/jobs/JLANDDA_POST_ANAL b/jobs/JLANDDA_POST_ANAL
@@ -76,8 +76,6 @@ mkdir -p ${COMOUT}
 export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
 mkdir -p ${DATA_SHARE}
 
-# Set other dates
-export NTIME=$($NDATE 24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/jobs/JLANDDA_PREP_OBS b/jobs/JLANDDA_PREP_OBS
@@ -78,8 +78,6 @@ mkdir -p ${COMOUTobs}
 export DATA_SHARE="${DATA_SHARE:-${DATAROOT}/DATA_SHARE/${PDY}}"
 mkdir -p ${DATA_SHARE}
 
-# Set other dates
-export PTIME=$($NDATE -24 $PDY$cyc)
 #
 #-----------------------------------------------------------------------
 #

diff --git a/modulefiles/build_container_intel.lua b/modulefiles/build_container_intel.lua
@@ -71,5 +71,5 @@ setenv("CC", "mpiicc")
 setenv("CXX", "mpiicpc")
 setenv("FC", "mpiifort")
 
-setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),""))
+setenv("JEDI_PATH", pathJoin(os.getenv("EPICHOME"),""))
 
diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua
@@ -47,4 +47,4 @@ setenv("FC", "mpiifort")
 setenv("CMAKE_Platform", "hera.intel")
 
 setenv("EPICHOME", "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev")
-setenv("JEDI_INSTALL", "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/jedi_v7")
+setenv("JEDI_PATH", "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/jedi_v7")
diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua
@@ -49,5 +49,5 @@ setenv("CXX", "mpiicpc")
 setenv("FC", "mpiifort")
 setenv("CMAKE_Platform", "hercules.intel")
 
-setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA_Dev")
-setenv("JEDI_INSTALL", "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules")
+setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA_v2.1")
+setenv("JEDI_PATH", "/work/noaa/epic/UFS_Land-DA_v2.1/jedi_v7_ic_hercules")
diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua
@@ -50,4 +50,4 @@ setenv("FC", "mpiifort")
 setenv("CMAKE_Platform", "orion.intel")
 
 setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA_Dev")
-setenv("JEDI_INSTALL", "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_stack1.6")
+setenv("JEDI_PATH", "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_stack1.6")
diff --git a/modulefiles/build_singularity_intel.lua b/modulefiles/build_singularity_intel.lua
@@ -100,6 +100,6 @@ setenv("CC", "mpiicc")
 setenv("CXX", "mpiicpc")
 setenv("FC", "mpiifort")
 
-setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),""))
+setenv("JEDI_PATH", pathJoin(os.getenv("EPICHOME"),""))
 
 whatis("Description: UFS build environment")
diff --git a/modulefiles/tasks/hera/task.prep_obs.lua b/modulefiles/tasks/hera/task.prep_obs.lua
@@ -1,7 +1,16 @@
-prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
+prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_unienv"))
 
 load(pathJoin("stack-intel", stack_intel_ver))
 load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("prod_util", prod_util_ver))
+
+load(pathJoin("py-cartopy", py_cartopy_ver))
+load(pathJoin("py-matplotlib", py_matplotlib_ver))
+load(pathJoin("py-netcdf4", py_netcdf4_ver))
+load(pathJoin("py-numpy", py_numpy_ver))
+load(pathJoin("py-pandas", py_pandas_ver))
+load(pathJoin("py-pyyaml", py_pyyaml_ver))
+load(pathJoin("py-xarray", py_xarray_ver))
+
diff --git a/parm/setup_wflow_env.py b/parm/setup_wflow_env.py
@@ -205,7 +205,8 @@ def set_default_parm():
         "fhrot": 0,
         "ic_data_model": "GFS",
         "imo": 384,
-        "jedi_install": "/path/to/jedi/install/dir",
+        "jedi_path": "/path/to/jedi/install/dir",
+        "jedi_py_ver": "/python/version/used/for/jedi",
         "jmo": 190,
         "lnd_calc_snet": ".true.",
         "lnd_ic_type": "custom",
@@ -220,8 +221,7 @@ def set_default_parm():
         "nprocs_analysis": 6,
         "nprocs_fcst_ic": 36,
         "obsdir": "",
-        "obsdir_subdir": "",
-        "obs_types": "GHCN",
+        "obs_ghcn": "YES",
         "output_fh": "1 -1",
         "res": 96,
         "restart_interval": "12 -1",
@@ -242,24 +242,29 @@ def set_machine_parm(machine):
     lowercase_machine = machine.lower()
     match lowercase_machine:
         case "hera":
-            jedi_install = "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/jedi_v7"
+            jedi_path = "/scratch2/NAGAPE/epic/UFS_Land-DA_Dev/jedi_v7"
+            jedi_py_ver = "python3.11"
             warmstart_dir = "/scratch2/NAGAPE/epic/UFS_Land-DA_v2.1/inputs/DATA_RESTART"
             max_cores_per_node = 40
         case "orion":
-            jedi_install = "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_stack1.6"
+            jedi_path = "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_stack1.6"
+            jedi_py_ver = "python3.10"
             warmstart_dir = "/work/noaa/epic/UFS_Land-DA_v2.1/inputs/DATA_RESTART"
             max_cores_per_node = 40
         case "hercules":
-            jedi_install = "/work/noaa/epic/UFS_Land-DA_Dev/jedi_v7_hercules"
+            jedi_path = "/work/noaa/epic/UFS_Land-DA_v2.1/jedi_v7_ic_hercules"
+            jedi_py_ver = "python3.10"
             warmstart_dir = "/work/noaa/epic/UFS_Land-DA_v2.1/inputs/DATA_RESTART"
             max_cores_per_node = 80
         case "singularity":
-            jedi_install = "SINGULARITY_WORKING_DIR"
+            jedi_path = "SINGULARITY_WORKING_DIR"
+            jedi_py_ver = "python3.11"
             warmstart_dir = "SINGULARITY_WORKING_DIR"
             max_cores_per_node = 40
 
     machine_config = {
-        "jedi_install": jedi_install,
+        "jedi_path": jedi_path,
+        "jedi_py_ver": jedi_py_ver,
         "warmstart_dir": warmstart_dir,
         "max_cores_per_node": max_cores_per_node,
     }

diff --git a/parm/templates/template.land_analysis.yaml b/parm/templates/template.land_analysis.yaml
@@ -25,7 +25,7 @@ workflow:
     CCPP_SUITE: "{{ ccpp_suite }}"
     COLDSTART: "{{ coldstart }}"
     COUPLER_CALENDAR: "{{ coupler_calendar }}"
-    DAtype: "letkfoi_snow"
+    DATE_CYCLE_FREQ_HR: "{{ date_cycle_freq_hr }}"
     DATE_FIRST_CYCLE: "{{ date_first_cycle }}"
     DT_ATMOS: "{{ dt_atmos }}"
     DT_RUNSEQ: "{{ dt_runseq }}"
@@ -35,7 +35,8 @@ workflow:
     FHROT: "{{ fhrot }}"
     IC_DATA_MODEL: "{{ ic_data_model }}"
     IMO: "{{ imo }}"
-    JEDI_INSTALL: "{{ jedi_install }}"
+    JEDI_PATH: "{{ jedi_path }}"
+    JEDI_PY_VER: "{{ jedi_py_ver }}"
     JMO: "{{ jmo }}"
     KEEPDATA: "YES"
     LND_CALC_SNET: "{{ lnd_calc_snet }}"
@@ -56,8 +57,7 @@ workflow:
     NPROCS_FORECAST_LND: "{{ nprocs_forecast_lnd }}"
     NPROCS_PER_NODE: "{{ nprocs_per_node }}"
     OBSDIR: "{{ obsdir }}"
-    OBSDIR_SUBDIR: "{{ obsdir_subdir }}"
-    OBS_TYPES: "{{ obs_types }}"
+    OBS_GHCN: "{{ obs_ghcn }}"
     OUTPUT_FH: "{{ output_fh }}"
     RES: "{{ res }}"
     RESTART_INTERVAL: "{{ restart_interval }}"
@@ -95,20 +95,22 @@ workflow:
         COMROOT: "&COMROOT;"
         cyc: "&cyc;"
         DATAROOT: "&DATAROOT;"
+        DATE_CYCLE_FREQ_HR: "&DATE_CYCLE_FREQ_HR;"
         HOMElandda: "&HOMElandda;"
+        JEDI_PATH: "&JEDI_PATH;"
+        JEDI_PY_VER: "&JEDI_PY_VER;"      
         KEEPDATA: "&KEEPDATA;"
         MACHINE: "&MACHINE;"
         model_ver: "&model_ver;"
         OBSDIR: "&OBSDIR;"
-        OBSDIR_SUBDIR: "&OBSDIR_SUBDIR;"
-        OBS_TYPES: "&OBS_TYPES;"
+        OBS_GHCN: "&OBS_GHCN;"
         PDY: "&PDY;"
         SCHED: "&SCHED;"
       account: "&ACCOUNT;"
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "prep_obs" "&HOMElandda;" "&MACHINE;"'
       jobname: prep_obs
       cores: 1
-      walltime: 00:02:00
+      walltime: 00:20:00
       queue: batch
       join: "&LOGDIR;/prep_obs&LOGFN_SUFFIX;"
 {%- if coldstart == "YES" %}
@@ -192,16 +194,16 @@ workflow:
         COMROOT: "&COMROOT;"
         COUPLER_CALENDAR: "&COUPLER_CALENDAR;"
         cyc: "&cyc;"
-        DAtype: "&DAtype;"
         DATAROOT: "&DATAROOT;"
+        DATE_CYCLE_FREQ_HR: "&DATE_CYCLE_FREQ_HR;"
         KEEPDATA: "&KEEPDATA;"
         HOMElandda: "&HOMElandda;"
-        JEDI_INSTALL: "&JEDI_INSTALL;"
+        JEDI_PATH: "&JEDI_PATH;"
         LOGDIR: "&LOGDIR;"
         MACHINE: "&MACHINE;"
         model_ver: "&model_ver;"
         NPROCS_ANALYSIS: "&NPROCS_ANALYSIS;"
-        OBS_TYPES: "&OBS_TYPES;"
+        OBS_GHCN: "&OBS_GHCN;"
         PDY: "&PDY;"
         RES: "&RES;"
         SCHED: "&SCHED;"
@@ -232,6 +234,7 @@ workflow:
         COMROOT: "&COMROOT;"
         cyc: "&cyc;"
         DATAROOT: "&DATAROOT;"
+        DATE_CYCLE_FREQ_HR: "&DATE_CYCLE_FREQ_HR;"
         FCSTHR: "&FCSTHR;"
         HOMElandda: "&HOMElandda;"
         KEEPDATA: "&KEEPDATA;"
@@ -272,9 +275,9 @@ workflow:
         COLDSTART: "&COLDSTART;"
         COMROOT: "&COMROOT;"
         cyc: "&cyc;"
-        DAtype: "&DAtype;"
         DATAROOT: "&DATAROOT;"
         DATE_FIRST_CYCLE: "&DATE_FIRST_CYCLE;"
+        DATE_CYCLE_FREQ_HR: "&DATE_CYCLE_FREQ_HR;"
         DT_ATMOS: "&DT_ATMOS;"
         DT_RUNSEQ: "&DT_RUNSEQ;"
         FCSTHR: "&FCSTHR;"
@@ -299,7 +302,6 @@ workflow:
         NPROCS_FORECAST_ATM: "&NPROCS_FORECAST_ATM;"
         NPROCS_FORECAST_LND: "&NPROCS_FORECAST_LND;"
         NPROCS_PER_NODE: "&NPROCS_PER_NODE;"
-        OBS_TYPES: "&OBS_TYPES;"
         OUTPUT_FH: "&OUTPUT_FH;"
         PDY: "&PDY;"
         RES: "&RES;"
@@ -339,6 +341,7 @@ workflow:
         COMROOT: "&COMROOT;"
         cyc: "&cyc;"
         DATAROOT: "&DATAROOT;"
+        DATE_CYCLE_FREQ_HR: "&DATE_CYCLE_FREQ_HR;"
         HOMElandda: "&HOMElandda;"
         KEEPDATA: "&KEEPDATA;"
         LOGDIR: "&LOGDIR;"