From c4f4c9868f11b1cfd5910f13d002d63b053ddbde Mon Sep 17 00:00:00 2001
From: Etienne Tourigny <etienne.tourigny@bsc.es>
Date: Thu, 1 Feb 2024 19:03:53 +0000
Subject: [PATCH 01/24] add new component co2box

---
 ece2cmor3/cmor_source.py           |  16 ++
 ece2cmor3/cmor_utils.py            |  22 ++
 ece2cmor3/co2box2cmor.py           | 439 +++++++++++++++++++++++++++++
 ece2cmor3/components.py            |   6 +-
 ece2cmor3/ece2cmor.py              |   2 +
 ece2cmor3/ece2cmorlib.py           |  12 +-
 ece2cmor3/resources/co2boxpar.json |  14 +
 7 files changed, 508 insertions(+), 3 deletions(-)
 create mode 100644 ece2cmor3/co2box2cmor.py
 create mode 100644 ece2cmor3/resources/co2boxpar.json

diff --git a/ece2cmor3/cmor_source.py b/ece2cmor3/cmor_source.py
index 60564ac8..8b5fdb69 100644
--- a/ece2cmor3/cmor_source.py
+++ b/ece2cmor3/cmor_source.py
@@ -46,6 +46,10 @@ def create_cmor_source(attributes, component):
         if src is None:
             log.error("Could not find a TM5 source variable within attributes %s" % (str(attributes.__dict__)))
         result = tm5_source(src)
+    if component == "co2box":
+        if src is None:
+            log.error("Could not find a co2box source variable within attributes %s" % (str(attributes.__dict__)))
+        result = co2box_source(src)
     # if component == NEWCOMPONENT:
     # create some source here, if NEWCOMPONENT has nc files as output, the NEMO case can be copied
     return result
@@ -273,3 +277,15 @@ def model_component(self):
 
     def variable(self):
         return self.colname_
+
+class co2box_source(cmor_source):
+
+    def __init__(self, colname):
+        super(co2box_source, self).__init__()
+        self.colname_ = colname
+
+    def model_component(self):
+        return "co2box"
+
+    def variable(self):
+        return self.colname_
diff --git a/ece2cmor3/cmor_utils.py b/ece2cmor3/cmor_utils.py
index f2f010c6..d7d63686 100644
--- a/ece2cmor3/cmor_utils.py
+++ b/ece2cmor3/cmor_utils.py
@@ -270,6 +270,28 @@ def get_tm5_interval(filepath):
         log.error("Date string in filename %s not supported." % fname)
     return start, end
 
+def find_co2box_output(path, expname, varname=None, freq=None):
+    """
+    Finds co2box outputfiles, which consist of varname_freq_expname_date.nc
+    inputs:
+    Path (mandatory)
+    expname (mandatory)
+    varname (optional)
+    freq (optional)
+    output:
+    list of full paths to files
+    """
+    if expname is None:
+        expname = "[a-zA-Z0-9]*"
+    if varname is None:
+        varname = "[a-zA-Z0-9]*"
+    if freq is None:
+        freq = "[a-zA-Z0-9]*"
+    date = "[0-9]{8}-[0-9]{8}"
+    rexp = "^%s_%s_%s_%s\.nc$"%(varname,freq,expname,date)
+    expr = re.compile(rexp)
+    a = [os.path.join(path, f) for f in os.listdir(path) if re.match(expr, f)]
+    return [os.path.join(path, f) for f in os.listdir(path) if re.match(expr, f)]
 
 # Writes the ncvar (numpy array or netcdf variable) to CMOR variable with id varid
 def netcdf2cmor(varid, ncvar, timdim=0, factor=1.0, term=0.0, psvarid=None, ncpsvar=None, swaplatlon=False,
diff --git a/ece2cmor3/co2box2cmor.py b/ece2cmor3/co2box2cmor.py
new file mode 100644
index 00000000..db844e55
--- /dev/null
+++ b/ece2cmor3/co2box2cmor.py
@@ -0,0 +1,439 @@
+import os
+import re
+import numpy
+import datetime
+import json
+import logging
+import netCDF4
+import cmor
+import cmor_utils
+import cmor_source
+import cmor_target
+import cmor_task
+import cdo
+from ece2cmor3 import cdoapi
+import Ngl
+import warnings
+from cdo import *
+
+# Logger object
+log = logging.getLogger(__name__)
+
+# Experiment name
+exp_name_ = None
+
+# Table root
+table_root_ = None
+
+# Files that are being processed in the current execution loop.
+co2box_files_ = []
+
+# Dictionary of co2box grid type with cmor grid id.
+dim_ids_ = {}
+
+# Dictionary of output frequencies with cmor time axis id.
+time_axes_ = {}
+
+time_axis_ids = {}
+
+# Reference date, times will be converted to hours since refdate
+ref_date_ = None
+
+unit_miss_match =[]
+failed  = []
+
+#
+#using_grid_=False
+path_=None
+# Initializes the processing loop.
+def initialize(path,expname,tabledir, prefix,refdate):
+    """initialize the cmorization for CO2BOX
+    Description:
+        
+    Input variables:
+        path, String: path to CO2BOX files
+        expname, string: name of the experiment
+        tabledir, string: path to tables
+        prefix, string: table prefix
+    Returns:
+        boolean: success
+    """
+    global log,co2box_files_,exp_name_,table_root_,ref_date_,path_
+    exp_name_ = expname
+    path_ = path
+    table_root_ =os.path.join(tabledir, prefix)
+    # select all CO2BOX files with expname from path
+    co2box_files_ = cmor_utils.find_co2box_output(path_,exp_name_)
+    if len(co2box_files_) == 0:
+        log.error('no CO2BOX varibles found, exiting!')
+        exit()
+    cal = None
+    ref_date_ = refdate
+
+    # read pressure level definitions from CMIP6_coordante file
+    # and save globally
+    coordfile = os.path.join(tabledir, prefix + "_coordinate.json")
+    if os.path.exists(coordfile):
+        with open(coordfile) as f:
+            data = json.loads(f.read())
+    else:
+        log.warning('Using default pressure level definitions')
+
+    cmor.load_table(table_root_ + "_grids.json")
+    return True
+
+
+# Resets the module globals.
+def finalize():
+    """finalize, clear variables
+    Args:
+        none
+    Retruns:
+        none
+    """
+    global co2box_files_,dim_ids_,time_axes_
+    log.info( 'Unit miss match variables %s '%(unit_miss_match))
+    co2box_files_ = []
+    dim_ids_ = {}
+    time_axes_ = {}
+
+# Executes the processing loop.
+def execute(tasks):
+    """execute the cmorization tasks for CO2BOX
+    Description:
+        
+    Args:
+        tasks (list): list of tasks 
+    Returns:
+        boolean: success
+    """
+    global log,time_axes_,table_root_,co2box_files_,using_grid_,path_,exp_name_
+    log.info("Executing %d co2box tasks..." % len(tasks))
+    log.info("Cmorizing co2box tasks...")
+    #Assign file to each task
+    for task in tasks:
+        setattr(task,cmor_task.output_path_key,None)
+        if not task.target.frequency in ['day', 'mon']:
+            task.set_failed()
+            log.info('Variable %s in table %s has frequency %s and thus not available in CO2BOX.'%(task.target.variable,task.target.table,task.target.frequency))
+            continue
+
+        # if the target frequency is monthly, create monthly file from daily output with "cdo monmean"
+        if task.target.frequency == 'mon':
+            co2box_files = cmor_utils.find_co2box_output(path_,exp_name_,task.target.variable,"day")
+            if len(co2box_files) != 1:
+                log.info('Did not find one file for variable %s in table %s from CO2BOX'%(task.target.variable,task.target.table))
+                task.set_failed()
+            else:
+                cdo = Cdo()
+                ifile=co2box_files[0]
+                ofile=ifile.replace("_day_","_mon_")
+                log.info('Creating temporary monthly mean file %s'%(ofile))
+                cdo.monmean(input=ifile, output=ofile)
+
+        co2box_files = cmor_utils.find_co2box_output(path_,exp_name_,task.target.variable,task.target.frequency)
+        if len(co2box_files) != 1:
+            log.info('Did not find one file for variable %s in table %s from CO2BOX'%(task.target.variable,task.target.table))
+            task.set_failed()
+        else:
+            setattr(task,cmor_task.output_path_key,co2box_files[0])
+
+    #group the taks according to table
+    taskdict = cmor_utils.group(tasks,lambda t:t.target.table)
+    for table,tasklist in taskdict.items():
+        try:
+            log.info("Loading CMOR table %s to process %d variables..." % (table,len(tasklist)))
+            tab_id = cmor.load_table("_".join([table_root_, table]) + ".json")
+            cmor.set_table(tab_id)
+        except Exception as e:
+            log.error("ERR -6: CMOR failed to load table %s, skipping variables %s. Reason: %s"
+                      % (table, ','.join([tsk.target.variable for tsk in tasklist]), e.message))
+            continue
+        dim_ids_['lat']=create_lat()
+        dim_ids_['lon']=create_lon()
+        # create or assign time axes to tasks
+        log.info("Creating time axes for table %s..." % table)
+        #create_time_axes(tasklist)
+        time_axes_=create_time_axes(tasklist)#time_axes
+
+        taskmask = dict([t,False] for t in tasklist)
+        for task in tasklist:
+            tmpfile = None
+            ncf=getattr(task,cmor_task.output_path_key)
+            tgtdims = getattr(task.target, cmor_target.dims_key).split()
+            if "latitude" in tgtdims and "longitude" in tgtdims:
+                setattr(task, 'lon', dim_ids_['lon'])
+                setattr(task, 'lat', dim_ids_['lat'])
+            if "site" in tgtdims:
+                log.critical('Z-dimension site not implemented ')
+                task.set_failed()
+                continue
+            if task.status==cmor_task.status_failed:
+                continue
+            if(taskmask[task] ):
+                log.warning("Ignoring source variable in nc file %s, since it has already been cmorized." % ncf)
+            else:
+                if task.status not in [cmor_task.status_failed]:
+                    log.info("Cmorizing source variable %s to target variable %s from file %s." % (task.source.variable(),task.target.variable,ncf))
+                    execute_netcdf_task(task,tab_id)
+                    if task.status<0:
+                        log.error("ERR -13: Cmorizing failed for %s" % (task.target.variable))
+                    else:
+                        taskmask[task] = True
+                        # remove temp file
+                        if task.target.frequency == 'mon' and os.path.basename(ncf).endswith('.nc.tmp'):
+                            os.unlink(ncf)
+                else:
+                    log.info("Skipping variable %s for unknown reason..." % (task.source.variable()))
+        for task,executed in taskmask.items():
+            if(not executed):
+                log.error("ERR -14: The source variable %s of target %s in  table %s failed to cmorize" % (task.source.variable(),task.target.variable,task.target.table))
+                failed.append([task.target.variable,task.target.table])
+    if len(unit_miss_match)>0:
+        log.info('Unit problems: %s'% unit_miss_match)
+    if len(failed)>0:
+        for ifail in failed:
+            log.info('Cmorization failed for : %s'%ifail)
+
+# Performs a single task.
+def execute_netcdf_task(task,tableid):
+    """excute task for netcdf data
+    Args:
+        task (cmor.task): task which will be handled
+        tableid (cmor.table): table which will have this task
+    Returns:
+        boolean: success of writing a variable
+    """
+    global log,dim_ids_,time_axes_
+    interpolate_to_pressure=False
+    task.status = cmor_task.status_cmorizing
+    filepath = getattr(task, cmor_task.output_path_key, None)
+
+    if not filepath:
+        log.error("ERR -15: Could not find file containing data for variable %s in table %s" % (task.target.variable,task.target.table))
+        task.set_failed()
+        return
+
+    store_var = getattr(task, "store_with", None)
+    if( task.target.dims >= 3):
+        if  ('lon' in dim_ids_ and 'lat' in dim_ids_):
+            axes = [dim_ids_['lat'],dim_ids_['lon']]
+        else:
+            dim_ids_['lat']=create_lat()
+            dim_ids_['lon']=create_lon()
+            axes=[dim_ids_['lat'],dim_ids_['lon']]
+        log.error('ERR -17: No z_axis_id found.')
+    elif ( task.target.dims == 2):
+        if not  hasattr(task, "z_axis_id"):
+            ''' 
+            2D variables lon+lat
+
+            '''
+            if not ('lon' in dim_ids_ and 'lat' in dim_ids_):
+                dim_ids_['lat']=create_lat()
+                dim_ids_['lon']=create_lon()
+                axes=[dim_ids_['lat'],dim_ids_['lon']]
+            else:
+                axes = [dim_ids_['lat'],dim_ids_['lon']]
+        else:
+            log.error('ERR -18: unsupported 2D dimensions %s'%task.target.dims)
+            exit('Exiting!')
+    elif task.target.dims==0:
+        axes=[]
+    else:
+        log.error('ERR -19: unsupported dimensions %s for variable'%(task.target.dims,task.target.variable))
+        exit()
+    time_id = getattr(task, "time_axis", 0)
+    if time_id != 0:
+        axes.append(time_id)
+    try:
+        dataset = netCDF4.Dataset(filepath, 'r')
+    except Exception as e:
+        log.error("ERR -20: Could not read netcdf file %s while cmorizing variable %s in table %s. Cause: %s" % (
+            filepath, task.target.variable, task.target.table, e.message))
+        return
+    varid = create_cmor_variable(task,dataset,axes)
+    if varid <0:
+        return False
+
+    ncvar = dataset.variables[task.source.variable()]
+
+    #handle normal case - global means
+    if True:# assumption: data is shape [time,lat,lon] (we need to roll longitude dimension so that 
+        #  the data corresponds to the dimension definition of tables (from [-180 , 180] to [0,360] deg).)
+        #  so by half the longitude dimension
+        missval = getattr(task.target, cmor_target.missval_key, 1.e+20)
+        # hack for co2s, we need the lowest model-level data only (assuming data is in form [time,level,lat,lon]
+        if task.target.variable == "co2s" and task.source.variable() == "co2":
+            vals=numpy.copy(ncvar[:,0,:,:])
+            # hack to convert units from mol/mol to ppm, cmor_task.conversion_key doesn't work fos some reason
+            if getattr(task.target,"units")=="1e-06" and getattr(ncvar,"units",None)=="mole mole-1":
+                vals=vals*1e6
+        else:
+            vals=numpy.copy(ncvar[:])
+        dims = numpy.shape(vals)
+        nroll=dims[-1]//2
+        ncvar = numpy.roll(vals,nroll,len(dims)-1)
+        vals=numpy.copy(ncvar[:,:,:])
+    # Default values
+    factor = 1.0
+    term=0.0
+    timdim=0
+    #check for missing values
+    if numpy.isnan(ncvar).any():
+        nanmask=~numpy.isnan(ncvar)
+    else:
+        nanmask=None
+    cmor_utils.netcdf2cmor(varid, ncvar, timdim, factor, term, store_var, None,
+                               swaplatlon=False, fliplat=True, mask=nanmask,missval=missval)
+    cmor.close(varid)
+
+    if store_var:
+        cmor.close(store_var)
+    task.status = cmor_task.status_cmorized
+   
+
+# Creates a variable in the cmor package
+def create_cmor_variable(task,dataset,axes):
+    """ Create cmor variable object
+    Args:
+        task (cmor.task): task for which we are creating a variable object
+        dataset (netcdf-dataset): netcdf dataset containing the data for CO2BOX for this variable
+        axes (list): list of axes ids for creation of cmor.variable object
+    Returns:
+        cmor.variable object: object identifier of created variable
+    
+    """
+    srcvar = task.source.variable()
+    ncvar = dataset.variables[srcvar]
+    unit = getattr(ncvar,"units",None)
+    if unit != getattr(task.target,"units"):
+        if unit=='mole mole-1':
+            # files have mole mole-1 but should be mol mol-1
+            unit = getattr(task.target,"units")
+        elif srcvar=='toz'or srcvar=='tropoz':
+            # unit is just different
+            if unit=='DU':
+                setattr(task,cmor_task.conversion_key,1e-5)
+            unit =  getattr(task.target,"units")
+        elif srcvar=='co2mass':
+            # co2mass gets converted to area sum
+            unit = getattr(task.target,"units")
+        else:
+            unit_miss_match.append(task.target.variable)
+            log.error("ERR -21: unit miss match, variable %s" % (task.target.variable))
+            return task.set_failed()
+
+    if((not unit) or hasattr(task,cmor_task.conversion_key)): # Explicit unit conversion
+        unit = getattr(task.target,"units")
+    if(hasattr(task.target,"positive") and len(task.target.positive) != 0):
+        return cmor.variable(table_entry = str(task.target.variable),units = str(unit),axis_ids = axes,original_name = str(srcvar),positive = "down")
+    else:
+        return cmor.variable(table_entry = str(task.target.variable),units = str(unit),axis_ids = axes,original_name = str(srcvar))
+
+# Creates time axes in cmor and attach the id's as attributes to the tasks
+def create_time_axes(tasks):
+    """ Create  time axes for all tasks
+    Args:
+        tasks (list): list of tasks for which time axes need to be created
+    Returns:
+        time_axes (dictionary): dictionary of time axes, with table+dimension combination as key
+    
+    """
+
+    global log#,time_axes_
+    time_axes = {}
+    for task in tasks:
+        freq=task.target.frequency
+        tgtdims = getattr(task.target, cmor_target.dims_key)
+        if getattr(task, cmor_task.output_path_key)==None:
+            continue
+        for time_dim in [d for d in list(set(tgtdims.split())) if d.startswith("time")]:
+            key=(task.target.table,time_dim)
+            if key in time_axes:
+                tid = time_axes[key]
+            else:
+                time_operator = getattr(task.target, "time_operator", ["point"])
+                log.info("Creating time axis using variable %s..." % task.target.variable)
+                tid = create_time_axis(path=getattr(task, cmor_task.output_path_key),
+                                       name=time_dim, has_bounds=(time_operator != ["point"]))
+                time_axes[key] = tid
+            setattr(task, "time_axis", tid)
+            break
+    return time_axes
+
+# Creates a tie axis for the corresponding table (which is suppoed to be loaded)
+def create_time_axis(path,name,has_bounds):
+    """ creage time axis for a give frequency
+    Args:
+        path (string): full path to netcdf file with this freq
+        name (string): tablename
+        has_bounds (boolean): true if it has bounds
+    Returns:
+        cmor.axis-object: time axis object with given freq
+    """
+    global log,ref_date_
+    vals = None
+    units = None
+    ds = None
+    #
+    ncfile=path
+    refdate = ref_date_
+    try:
+        ds = netCDF4.Dataset(ncfile)
+        timvar = ds.variables["time"]
+        co2boxunit = ds.variables["time"].units
+        vals = timvar[:]
+        units = getattr(timvar,"units")
+        if has_bounds:
+            bnds = getattr(timvar,"bounds")
+            bndvar = ds.variables[bnds]
+    except:
+        ds.close()
+    co2boxrefdate=datetime.datetime.strptime(co2boxunit,"days since %Y-%m-%d %H:%M:%S")
+    # delta days for change of reftime
+    diff_days= (refdate-co2boxrefdate).total_seconds()/86400
+    vals=vals-diff_days
+    if has_bounds:
+        bndvar2=numpy.zeros_like(bndvar)
+        bndvar2[:,0]=bndvar[:,0]-int(diff_days)
+        bndvar2[:,1]=bndvar[:,1]-int(diff_days)
+        bndvar=bndvar2
+        # hourly bounds can have overlap by -1e-14, which is caught by cmor library as an error
+        # so just correct it always
+        for i in range(numpy.shape(bndvar2)[0]-1):
+            bndvar2[i+1,0]=bndvar2[i,1]
+    if(len(vals) == 0 or units == None):
+        log.error("ERR -22: No time values or units could be read from co2box output files %s" % str(path))
+        return -1
+    units="days since " + str(ref_date_)
+    ####
+    if has_bounds:
+        return cmor.axis(table_entry = str(name), units=units, coord_vals = vals,cell_bounds = bndvar[:,:])
+    else:
+        return cmor.axis(table_entry = str(name), units=units, coord_vals = vals)
+
+      
+def create_lat():
+    """Create latitude dimension
+    Args:
+        none
+    Returns:
+        lat_id (cmor.axis): cmor.axis-object 
+    """
+    lat_id=cmor.axis(table_entry="latitude", units="degrees_north",
+                                    coord_vals=[0.], cell_bounds=[0.,0.])
+    return lat_id
+
+def create_lon():
+    """Create longitude dimension
+    Args:
+        none
+    Returns:
+        lon_id (cmor_axis): cmor.axis-object 
+    """
+    lon_id=cmor.axis(table_entry="longitude", units="degrees_east",
+                                    coord_vals=[0.], cell_bounds=[0.,0.])
+    return lon_id
+    
+
diff --git a/ece2cmor3/components.py b/ece2cmor3/components.py
index e3c95cd4..1d88e341 100644
--- a/ece2cmor3/components.py
+++ b/ece2cmor3/components.py
@@ -16,13 +16,15 @@
           "lpjg": {realms: ["land", "atmos"],
                    table_file: os.path.join(os.path.dirname(__file__), "resources", "lpjgpar.json")},
           "tm5": {realms: ["aerosol", "atmosChem", "atmos"],
-                  table_file: os.path.join(os.path.dirname(__file__), "resources", "tm5par.json")}
+                  table_file: os.path.join(os.path.dirname(__file__), "resources", "tm5par.json")},
+          "co2box": {realms: ["atmosChem"],
+                  table_file: os.path.join(os.path.dirname(__file__), "resources", "co2boxpar.json")}
           }
 
 ece_configs = {'EC-EARTH-AOGCM'   : ["ifs", "nemo"               ],
                'EC-EARTH-HR'      : ["ifs", "nemo"               ],
                'EC-EARTH-LR'      : ["ifs", "nemo"               ],
-               'EC-EARTH-CC'      : ["ifs", "nemo", "tm5", "lpjg"],
+               'EC-EARTH-CC'      : ["ifs", "nemo", "tm5", "lpjg", "co2box"],
                'EC-EARTH-GrisIS'  : ["ifs", "nemo"               ], # If a PISM component is added to ece2cmor3 it needs here to be added as well.
                'EC-EARTH-AerChem' : ["ifs", "nemo", "tm5"        ],
                'EC-EARTH-Veg'     : ["ifs", "nemo", "lpjg"       ],
diff --git a/ece2cmor3/ece2cmor.py b/ece2cmor3/ece2cmor.py
index c55dfef0..e6c64f21 100755
--- a/ece2cmor3/ece2cmor.py
+++ b/ece2cmor3/ece2cmor.py
@@ -146,6 +146,8 @@ def ifs_model_level_variable(target):
         ece2cmorlib.perform_lpjg_tasks(args.datadir, args.tmpdir, args.exp, refdate)
     if "tm5" in active_components:
         ece2cmorlib.perform_tm5_tasks(args.datadir, args.tmpdir, args.exp, refdate)
+    if "co2box" in active_components:
+        ece2cmorlib.perform_co2box_tasks(args.datadir, args.tmpdir, args.exp, refdate)
 
 #   if procNEWCOMPONENT in active_components:
 #       ece2cmorlib.perform_NEWCOMPONENT_tasks(args.datadir, args.exp, refdate)
diff --git a/ece2cmor3/ece2cmorlib.py b/ece2cmor3/ece2cmorlib.py
index f9605f2f..ae81a9bf 100644
--- a/ece2cmor3/ece2cmorlib.py
+++ b/ece2cmor3/ece2cmorlib.py
@@ -9,7 +9,7 @@
 import cdo       # Only for version printing
 import dreqPy    # Only for version printing
 
-from ece2cmor3 import __version__, cmor_target, cmor_task, nemo2cmor, ifs2cmor, lpjg2cmor, tm52cmor, postproc, \
+from ece2cmor3 import __version__, cmor_target, cmor_task, nemo2cmor, ifs2cmor, lpjg2cmor, tm52cmor, co2box2cmor, postproc, \
     cmor_utils, cmor_source
 
 # Logger instance
@@ -255,6 +255,16 @@ def perform_tm5_tasks(datadir, ncdir, expname, refdate=None):
         return
     tm52cmor.execute(tm5_tasks)
 
+# Performs a co2box cmorization processing:
+def perform_co2box_tasks(datadir, ncdir, expname, refdate=None):
+    global log, tasks, table_dir, prefix
+    validate_setup_settings()
+    validate_run_settings(datadir, expname)
+    co2box_tasks = [t for t in tasks if t.source.model_component() == "co2box"]
+    log.info("Selected %d co2box tasks from %d input tasks" % (len(co2box_tasks), len(tasks)))
+    if (not co2box2cmor.initialize(datadir, expname, table_dir, prefix, refdate)):
+        return
+    co2box2cmor.execute(co2box_tasks)
 
 # def perform_NEWCOMPONENT_tasks(datadir, expname, startdate, interval):
 #    global log, tasks, table_dir, prefix
diff --git a/ece2cmor3/resources/co2boxpar.json b/ece2cmor3/resources/co2boxpar.json
new file mode 100644
index 00000000..741c4e1e
--- /dev/null
+++ b/ece2cmor3/resources/co2boxpar.json
@@ -0,0 +1,14 @@
+[
+    {
+        "source": "co2s",
+        "target": "co2s"
+    },
+    {
+        "source": "co2",
+        "target": "co2s"
+    },
+    {
+        "source": "co2mass",
+        "target": "co2mass"
+    }
+]

From 4aecbc086becae8ed0a7bfa6a51479ccb0a92c43 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 2 Feb 2024 00:10:41 +0100
Subject: [PATCH 02/24] Add the non-cmor co2box variables co2mass & co2s #785
 #792.

---
 ece2cmor3/scripts/add-variables-for-co2box.sh | 92 +++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100755 ece2cmor3/scripts/add-variables-for-co2box.sh

diff --git a/ece2cmor3/scripts/add-variables-for-co2box.sh b/ece2cmor3/scripts/add-variables-for-co2box.sh
new file mode 100755
index 00000000..81463d15
--- /dev/null
+++ b/ece2cmor3/scripts/add-variables-for-co2box.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+# Thomas Reerink
+#
+# This script adds some non-cmor variables (which thus do not exit in
+# the CMIP6 data request) to the Eyr cmor table.
+#
+# This script requires no arguments.
+#
+# Run example:
+#  ./add-variables-for-co2box.sh
+#
+
+if [ "$#" -eq 0 ]; then
+
+ add_variables_for_co2box=True
+
+ if [ add_variables_for_co2box ]; then
+  # See #785  https://github.com/EC-Earth/ece2cmor3/issues/785
+  # See #792  https://github.com/EC-Earth/ece2cmor3/pull/792
+
+  # co2mass
+  # co2s
+
+  table_path=../resources/cmip6-cmor-tables/Tables/
+  table_file_day=CMIP6_day.json
+
+  cd ${table_path}
+  git checkout ${table_file_day}
+
+  sed -i  '/"hfls": {/i \
+        "co2mass": {                                                   \
+            "frequency": "day",                                        \
+            "modeling_realm": "atmos",                                 \
+            "standard_name": "atmosphere_mass_of_carbon_dioxide",      \
+            "units": "kg",                                             \
+            "cell_methods": "area: time: mean",                        \
+            "cell_measures": "",                                       \
+            "long_name": "Total Atmospheric Mass of CO2",              \
+            "comment": "Total atmospheric mass of Carbon Dioxide",     \
+            "dimensions": "time",                                      \
+            "out_name": "co2mass",                                     \
+            "type": "real",                                            \
+            "positive": "",                                            \
+            "valid_min": "",                                           \
+            "valid_max": "",                                           \
+            "ok_min_mean_abs": "",                                     \
+            "ok_max_mean_abs": ""                                      \
+        },                                                             \
+       "co2s": {                                                       \
+            "frequency": "day",                                        \
+            "modeling_realm": "atmos",                                 \
+            "standard_name": "mole_fraction_of_carbon_dioxide_in_air", \
+            "units": "1e-06",                                          \
+            "cell_methods": "time: mean",                              \
+            "cell_measures": "area: areacella",                        \
+            "long_name": "Atmosphere CO2",                             \
+            "comment": "As co2, but only at the surface",              \
+            "dimensions": "longitude latitude time",                   \
+            "out_name": "co2s",                                        \
+            "type": "real",                                            \
+            "positive": "",                                            \
+            "valid_min": "",                                           \
+            "valid_max": "",                                           \
+            "ok_min_mean_abs": "",                                     \
+            "ok_max_mean_abs": ""                                      \
+        },
+  ' ${table_file_day}
+
+  # Remove the trailing spaces of the inserted block above:
+  sed -i -e 's/\s*$//g' -e 's/,$/, /g' ${table_file_day}
+
+  cd -
+
+  echo
+  echo " $0 reports:"
+  echo "  The adjusted file is:  ${table_path}/${table_file_day}"
+  echo "  Which is part of a nested repository, therefore to view the diff, run:"
+  echo "  cd ${table_path}; git diff; cd -"
+  echo
+
+ else
+    echo
+    echo " Nothing done, no set of variables and / or experiments has been selected to add to the tables."
+    echo
+ fi
+
+else
+    echo
+    echo " This scripts requires no argument:"
+    echo "  $0"
+    echo
+fi

From 621c2df5d36e24243c2008109b3d66b9fb2a6cb0 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Mon, 5 Feb 2024 13:15:45 +0100
Subject: [PATCH 03/24] Fix with which pytest locally comes through without
 errors #792 #785.

---
 ece2cmor3/co2box2cmor.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/ece2cmor3/co2box2cmor.py b/ece2cmor3/co2box2cmor.py
index db844e55..82a1769a 100644
--- a/ece2cmor3/co2box2cmor.py
+++ b/ece2cmor3/co2box2cmor.py
@@ -6,15 +6,11 @@
 import logging
 import netCDF4
 import cmor
-import cmor_utils
-import cmor_source
-import cmor_target
-import cmor_task
 import cdo
-from ece2cmor3 import cdoapi
 import Ngl
 import warnings
 from cdo import *
+from ece2cmor3 import cdoapi, cmor_source, cmor_target, cmor_task, cmor_utils, postproc
 
 # Logger object
 log = logging.getLogger(__name__)

From c42d550de0727b816a0c8b14261df3a14bfe8cd9 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Mon, 5 Feb 2024 13:55:59 +0100
Subject: [PATCH 04/24] Add the co2box variables to the request and include
 them in the optimism genecec flow #792 #785.

---
 .../optimesm-request-EC-EARTH-CC-varlist.json      | 14 +++++++++++++-
 ece2cmor3/scripts/add-variables-for-co2box.sh      |  3 ++-
 .../scripts/genecec-for-individual-experiments.sh  |  1 +
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json b/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
index 05a6759f..3ce7e181 100644
--- a/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
+++ b/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
@@ -492,5 +492,17 @@
             "snmassacrossline"
         ]
     },
-    "tm5": {}
+    "tm5": {},
+    "co2box": {
+        "Amon": [
+            "co2mass"
+        ],
+        "Emon": [
+            "co2s"
+        ],
+        "day": [
+            "co2s",
+            "co2mass"
+        ]
+    }
 }
diff --git a/ece2cmor3/scripts/add-variables-for-co2box.sh b/ece2cmor3/scripts/add-variables-for-co2box.sh
index 81463d15..df8755a3 100755
--- a/ece2cmor3/scripts/add-variables-for-co2box.sh
+++ b/ece2cmor3/scripts/add-variables-for-co2box.sh
@@ -73,7 +73,8 @@ if [ "$#" -eq 0 ]; then
 
   echo
   echo " $0 reports:"
-  echo "  The adjusted file is:  ${table_path}/${table_file_day}"
+  echo "  The adjusted file is:"
+  echo "   ${table_path}/${table_file_day}"
   echo "  Which is part of a nested repository, therefore to view the diff, run:"
   echo "  cd ${table_path}; git diff; cd -"
   echo
diff --git a/ece2cmor3/scripts/genecec-for-individual-experiments.sh b/ece2cmor3/scripts/genecec-for-individual-experiments.sh
index 4ee07a72..e46167b1 100755
--- a/ece2cmor3/scripts/genecec-for-individual-experiments.sh
+++ b/ece2cmor3/scripts/genecec-for-individual-experiments.sh
@@ -79,6 +79,7 @@ if [ "$#" -eq 5 ]; then
   # optimesm only:
   if [ ${data_request_file##*/} = 'optimesm-request-EC-EARTH-CC-varlist.json' ]; then
    ./add-lpjg-cc-diagnostics.sh
+   ./add-variables-for-co2box.sh
   fi
 
   rm -rf   ${output_dir}

From 6c259352c70baaabf26a285e4eaa244c5b3e5fa2 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 9 Feb 2024 11:40:45 +0100
Subject: [PATCH 05/24] minor: Align #792.

---
 ece2cmor3/scripts/add-variables-for-co2box.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ece2cmor3/scripts/add-variables-for-co2box.sh b/ece2cmor3/scripts/add-variables-for-co2box.sh
index df8755a3..5da55de2 100755
--- a/ece2cmor3/scripts/add-variables-for-co2box.sh
+++ b/ece2cmor3/scripts/add-variables-for-co2box.sh
@@ -46,7 +46,7 @@ if [ "$#" -eq 0 ]; then
             "ok_min_mean_abs": "",                                     \
             "ok_max_mean_abs": ""                                      \
         },                                                             \
-       "co2s": {                                                       \
+        "co2s": {                                                      \
             "frequency": "day",                                        \
             "modeling_realm": "atmos",                                 \
             "standard_name": "mole_fraction_of_carbon_dioxide_in_air", \

From a13ee94fba0945d00bce13c0c81f5314c74cba7f Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 9 Feb 2024 16:29:17 +0100
Subject: [PATCH 06/24] Add the co2box component (for request-overview file) in
 the genecec chain #785 #792.

---
 .../scripts/convert_component_to_flat_json.py | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/ece2cmor3/scripts/convert_component_to_flat_json.py b/ece2cmor3/scripts/convert_component_to_flat_json.py
index b85e3758..86d47ef2 100755
--- a/ece2cmor3/scripts/convert_component_to_flat_json.py
+++ b/ece2cmor3/scripts/convert_component_to_flat_json.py
@@ -2,7 +2,7 @@
 # Thomas Reerink
 #
 # This script converts a component json file produced by genecec including the preferences
-# into a flat json without the component structue (ifs, nemo, lpjg & tm5) such that it can be
+# into a flat json without the component structue (ifs, nemo, lpjg, tm5 & co2box) such that it can be
 # read with: checkvars -v --asciionly --drq flat-json-file.json --output request-overview
 #
 # Note that when a flat json file is given (instead of a component json file) the produced json
@@ -45,6 +45,8 @@ def main():
         nemo_request = data_request["nemo"]
         lpjg_request = data_request["lpjg"]
         tm5_request  = data_request["tm5"]
+        co2box_request = data_request["co2box"]
+       #NEWCOMPONENT_request = data_request["NEWCOMPONENT"]
 
 
         # Determine whether a same table is present in the nemo dictionary as in the ifs dictionary:
@@ -71,6 +73,22 @@ def main():
          else:
           ifs_request.update({x: tm5_request[x]})
 
+        # Determine whether a same table is present in the co2box dictionary as in the ifs dictionary:
+        for x in co2box_request:
+         if x in ifs_request:
+          for i in range(0, len(co2box_request[x])):
+           ifs_request[x].append(co2box_request[x][i])
+         else:
+          ifs_request.update({x: co2box_request[x]})
+
+       ## Determine whether a same table is present in the NEWCOMPONENT dictionary as in the ifs dictionary:
+       #for x in NEWCOMPONENT_request:
+       # if x in ifs_request:
+       #  for i in range(0, len(NEWCOMPONENT_request[x])):
+       #   ifs_request[x].append(NEWCOMPONENT_request[x][i])
+       # else:
+       #  ifs_request.update({x: NEWCOMPONENT_request[x]})
+
         with open(output_json_file, 'w') as outfile:
             json.dump(ifs_request, outfile, sort_keys=True, indent=4)
         outfile.close()

From d58295b606cbf9e5f3c6742ea74bf6093c56f846 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 9 Feb 2024 16:30:59 +0100
Subject: [PATCH 07/24] Fix multiple source error #785 792.

---
 ece2cmor3/resources/co2boxpar.json | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/ece2cmor3/resources/co2boxpar.json b/ece2cmor3/resources/co2boxpar.json
index 741c4e1e..870bcc55 100644
--- a/ece2cmor3/resources/co2boxpar.json
+++ b/ece2cmor3/resources/co2boxpar.json
@@ -3,10 +3,6 @@
         "source": "co2s",
         "target": "co2s"
     },
-    {
-        "source": "co2",
-        "target": "co2s"
-    },
     {
         "source": "co2mass",
         "target": "co2mass"

From 8056eac18cb19e0ea60f0af91ff7079d83fc0be3 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Mon, 12 Feb 2024 11:51:10 +0100
Subject: [PATCH 08/24] Adding LPJGmon fco2nat for saving the pure LPJG field
 as agreed in #798 #785.

---
 .../optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json   | 1 +
 ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh                     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json b/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
index c40751aa..f307e725 100644
--- a/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
+++ b/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
@@ -288,6 +288,7 @@
             "evspsbl",
             "evspsblpot",
             "evspsblsoi",
+            "fco2nat",
             "mrroLut",
             "mrsll",
             "mrsol",
diff --git a/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh b/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
index 2a4c69f9..a7c8bc94 100755
--- a/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
+++ b/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
@@ -202,6 +202,7 @@ if [ "$#" -eq 0 ]; then
   grep -A 17 -e '"evspsbl":'    CMIP6_Amon.json                            >> ${table_file_LPJGmon}
   grep -A 17 -e '"evspsblpot":' CMIP6_Emon.json                            >> ${table_file_LPJGmon}
   grep -A 17 -e '"evspsblsoi":' CMIP6_Lmon.json                            >> ${table_file_LPJGmon}
+  grep -A 17 -e '"fco2nat":'    CMIP6_Amon.json                            >> ${table_file_LPJGmon}
   grep -A 17 -e '"mrroLut":'    CMIP6_Emon.json                            >> ${table_file_LPJGmon}
   grep -A 17 -e '"mrsll":'      CMIP6_Emon.json                            >> ${table_file_LPJGmon}
   grep -A 17 -e '"mrsol":'      CMIP6_Emon.json                            >> ${table_file_LPJGmon}

From b10eaf6202e7325134839574e3ff8b1ba3d2385f Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Wed, 14 Feb 2024 21:26:58 +0100
Subject: [PATCH 09/24] Fix preference issue with recently added LPJG variable
 variants #801.

---
 ece2cmor3/resources/prefs.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ece2cmor3/resources/prefs.py b/ece2cmor3/resources/prefs.py
index 351a0b4f..3661c4dc 100644
--- a/ece2cmor3/resources/prefs.py
+++ b/ece2cmor3/resources/prefs.py
@@ -25,8 +25,9 @@ def keep_variable(target, model_component, ecearth_config):
         else:
             return model_component == "ifs"
 
-    # Soil moisture etc: prefer ifs over lpjguess in all cases (?)
-    if variable in ["mrso", "mrsol", "mrsos", "mrro", "mrros", "tsl"]:
+    # All watercycle related variables and Temperature of Soil (tsl) from IFS/HTESSEL have preference above the ones from LPJG
+    # because they form a consistant set with other IFS variables:
+    if variable in ["evspsbl", "mrfso", "mrso", "mrsol", "mrsos", "mrro", "mrros", "snc", "snd", "snw", "tsl"]:
         return model_component == "ifs"
 
     # Carbon-cycle variables only activated in EC-EARTH-CC

From 15f37423c8bb390e9b82d6830334905c98489b18 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Wed, 14 Feb 2024 21:29:03 +0100
Subject: [PATCH 10/24] Remove earlier warning for potential preferrence issue
 with these recently added LPJG variable variants #801.

---
 ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh b/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
index a7c8bc94..5c52c472 100755
--- a/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
+++ b/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
@@ -10,13 +10,6 @@
 #  ./add-lpjg-cc-diagnostics.sh
 #
 
-# Potential preference issues after adding the LPJG table variables (to be checked and tested):
-# Amon        evspsbl       ifs  code name = 182.128 | lpjg code name = evspsbl
-# Lmon        mrfso         ifs  code name =  85.129, expression = 1000*(0.07*var39*((var139<270.16)+0.5*(var139<274.16 && var139>270.16)*(1-sin(0.785*(var139-272.16)))) + 0.21*var40*((var170<270.16)+0.5*(var170<274.16 && var170>270.16)*(1-sin(0.785*(var170-272.16)))) + 0.72*var41*((var183<270.16)+0.5*(var183<274.16 && var183>270.16)*(1-sin(0.785*(var183-272.16)))) + 1.89*var42*((var236<270.16)+0.5*(var236<274.16 && var236>270.16)*(1-sin(0.785*(var236-272.16))))) | lpjg code name = mrfso
-# LImon       snc           ifs  code name = 119.129, expression = 100*(var141>0) | lpjg code name = snc
-# LImon       snd           ifs  code name = 115.129, expression = 1000*var141/var33 | lpjg code name = snd
-# LImon       snw           ifs  code name = 141.128 | lpjg code name = snw
-
 if [ "$#" -eq 0 ]; then
 
  add_the_lpjg_cc_diagnostics=True

From 86a418b7abe903b30523603014c84c0bed62a93d Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Thu, 15 Feb 2024 16:48:15 +0100
Subject: [PATCH 11/24] Add the ece2cmor3 revision and git short & long hash to
 the log.info.

---
 ece2cmor3/ece2cmorlib.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/ece2cmor3/ece2cmorlib.py b/ece2cmor3/ece2cmorlib.py
index f9605f2f..3d58e0c4 100644
--- a/ece2cmor3/ece2cmorlib.py
+++ b/ece2cmor3/ece2cmorlib.py
@@ -4,7 +4,7 @@
 import json
 import logging
 import os
-#import subprocess
+import subprocess
 import tempfile
 import cdo       # Only for version printing
 import dreqPy    # Only for version printing
@@ -42,6 +42,11 @@
 PRESERVE = cmor.CMOR_PRESERVE
 PRESERVE_NC3 = cmor.CMOR_PRESERVE_3
 
+def get_git_revision_hash() -> str:
+    return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip()
+
+def get_git_revision_short_hash() -> str:
+    return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode('ascii').strip()
 
 # Initialization function without using the cmor library, must be called before starting
 def initialize_without_cmor(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default,
@@ -49,6 +54,7 @@ def initialize_without_cmor(metadata_path=conf_path_default, mode=cmor_mode_defa
     global prefix, table_dir, targets, metadata, cmor_mode
     with open(metadata_path, 'r') as f:
         metadata = json.load(f)
+    log.info('ece2cmor3 version {} with ece2cmor_git_revision {} with the long hash: {}'.format(__version__.version, get_git_revision_short_hash(), get_git_revision_hash()))
     log.info('Python {:} {:}'.format(os.sys.version[0:68], os.sys.version[69:80]))
    #log.info('Python version info: {:}'.format(os.sys.version_info))
     log.info('cdo {:}'.format(cdo.Cdo().version()))
@@ -71,6 +77,7 @@ def initialize(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir
     global prefix, table_dir, targets, metadata, cmor_mode
     with open(metadata_path, 'r') as f:
         metadata = json.load(f)
+    log.info('ece2cmor3 version {} with ece2cmor_git_revision {} with the long hash: {}'.format(__version__.version, get_git_revision_short_hash(), get_git_revision_hash()))
     log.info('Python {:} {:}'.format(os.sys.version[0:68], os.sys.version[69:80]))
    #log.info('Python version info: {:}'.format(os.sys.version_info))
     log.info('cdo {:}'.format(cdo.Cdo().version()))

From a2003abb2a86ee2052d54eace47909c40184b840 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Thu, 15 Feb 2024 16:49:53 +0100
Subject: [PATCH 12/24] Clean: a never used preference file for a different
 design.

---
 ece2cmor3/taskloader.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ece2cmor3/taskloader.py b/ece2cmor3/taskloader.py
index 5e198ff8..00cd8d6e 100644
--- a/ece2cmor3/taskloader.py
+++ b/ece2cmor3/taskloader.py
@@ -20,8 +20,6 @@
 json_script_key   = "script"
 json_src_key      = "src"
 
-variable_prefs_file          = os.path.join(os.path.dirname(__file__), "resources",                               "varprefs.csv")
-
 omit_vars_file_01            = os.path.join(os.path.dirname(__file__), "resources", "lists-of-omitted-variables", "list-of-omitted-variables-01.xlsx")
 omit_vars_file_02            = os.path.join(os.path.dirname(__file__), "resources", "lists-of-omitted-variables", "list-of-omitted-variables-02.xlsx")
 omit_vars_file_03            = os.path.join(os.path.dirname(__file__), "resources", "lists-of-omitted-variables", "list-of-omitted-variables-03.xlsx")

From e6ba2874f8a9918566c3512b40c71230aaa6f3ba Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Thu, 15 Feb 2024 16:50:57 +0100
Subject: [PATCH 13/24] Add the NEWCOMPONENT instruction keyword for the
 metadata part.

---
 ece2cmor3/scripts/modify-metadata-template.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ece2cmor3/scripts/modify-metadata-template.sh b/ece2cmor3/scripts/modify-metadata-template.sh
index 7d867e6d..8aa9af5a 100755
--- a/ece2cmor3/scripts/modify-metadata-template.sh
+++ b/ece2cmor3/scripts/modify-metadata-template.sh
@@ -50,6 +50,7 @@ if [ "$#" -eq 3 ] || [ "$#" -eq 4 ]; then
   source_type_index=7  # The usual AOGCM case
  fi
 
+ # Add your NEWCOMPONENT to this if-blocks if you want to extend ece2cmor3 for more model components.
  if [ "${ececonf}" = 'EC-EARTH-AOGCM'   ]; then declare -a model_components=('ifs' 'nemo'             ); fi
  if [ "${ececonf}" = 'EC-EARTH-HR'      ]; then declare -a model_components=('ifs' 'nemo'             ); fi
  if [ "${ececonf}" = 'EC-EARTH-LR'      ]; then declare -a model_components=('ifs' 'nemo'             ); fi
@@ -434,11 +435,13 @@ if [ "$#" -eq 3 ] || [ "$#" -eq 4 ]; then
      echo ' Running ' $0 ${mip} ${experiment} ${ececonf}                   '  for ' ${component}
     fi
 
+    # Add your NEWCOMPONENT to this check if you want to extend ece2cmor3 for more model components.
     if [ "${component}" != 'ifs' ] && [ "${component}" != 'nemo' ] && [ "${component}" != 'tm5' ] && [ "${component}" != 'lpjg' ]; then
      echo ' Error in ' $0 ': unkown ec-earth component: '  ${component} '  Valid options: ifs, nemo, tm5, or lpjg'
      exit
     fi
 
+    # Add your NEWCOMPONENT to this if-block if you want to extend ece2cmor3 for more model components.
     if [ "${component}" = 'ifs' ]; then
      grid_label='gr'
      res_index=1

From f12accb137f31beb74ffc766c86ad622f88a9d71 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Thu, 15 Feb 2024 16:53:33 +0100
Subject: [PATCH 14/24] Add message for used case.

---
 ece2cmor3/versioncheck.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ece2cmor3/versioncheck.py b/ece2cmor3/versioncheck.py
index e787a946..f385afa7 100755
--- a/ece2cmor3/versioncheck.py
+++ b/ece2cmor3/versioncheck.py
@@ -19,6 +19,7 @@ def main(args=None):
     # Set your test case:
     case=2
 
+    print('\nTest case {}:'.format(case))
     if case == 1:
      from setup import get_git_hash
      parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__.version + ' with sha: ' + get_git_hash())

From ae2b16562e1186a513f1300126ffe62d1542c71f Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Thu, 15 Feb 2024 22:22:41 +0100
Subject: [PATCH 15/24] Add branch name in log.info and separate ece2cmor3
 version and git hash logging.

---
 ece2cmor3/ece2cmorlib.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/ece2cmor3/ece2cmorlib.py b/ece2cmor3/ece2cmorlib.py
index 3d58e0c4..9be475d2 100644
--- a/ece2cmor3/ece2cmorlib.py
+++ b/ece2cmor3/ece2cmorlib.py
@@ -48,13 +48,17 @@ def get_git_revision_hash() -> str:
 def get_git_revision_short_hash() -> str:
     return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD']).decode('ascii').strip()
 
+def get_git_branch_name() -> str:
+    return subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('ascii').strip()
+
 # Initialization function without using the cmor library, must be called before starting
 def initialize_without_cmor(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir=table_dir_default,
                             tableprefix=prefix_default):
     global prefix, table_dir, targets, metadata, cmor_mode
     with open(metadata_path, 'r') as f:
         metadata = json.load(f)
-    log.info('ece2cmor3 version {} with ece2cmor_git_revision {} with the long hash: {}'.format(__version__.version, get_git_revision_short_hash(), get_git_revision_hash()))
+    log.info('ece2cmor3 version {} based on the {} branch'.format(__version__.version, get_git_branch_name()))
+    log.info('ece2cmor_git_revision {} with the long hash: {}'.format(get_git_revision_short_hash(), get_git_revision_hash()))
     log.info('Python {:} {:}'.format(os.sys.version[0:68], os.sys.version[69:80]))
    #log.info('Python version info: {:}'.format(os.sys.version_info))
     log.info('cdo {:}'.format(cdo.Cdo().version()))
@@ -77,7 +81,8 @@ def initialize(metadata_path=conf_path_default, mode=cmor_mode_default, tabledir
     global prefix, table_dir, targets, metadata, cmor_mode
     with open(metadata_path, 'r') as f:
         metadata = json.load(f)
-    log.info('ece2cmor3 version {} with ece2cmor_git_revision {} with the long hash: {}'.format(__version__.version, get_git_revision_short_hash(), get_git_revision_hash()))
+    log.info('ece2cmor3 version {} based on the {} branch'.format(__version__.version, get_git_branch_name()))
+    log.info('ece2cmor_git_revision {} with the long hash: {}'.format(get_git_revision_short_hash(), get_git_revision_hash()))
     log.info('Python {:} {:}'.format(os.sys.version[0:68], os.sys.version[69:80]))
    #log.info('Python version info: {:}'.format(os.sys.version_info))
     log.info('cdo {:}'.format(cdo.Cdo().version()))

From 487a58fc8206d2dd377b7eb1c2d0c0b358348f2d Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Thu, 15 Feb 2024 22:35:29 +0100
Subject: [PATCH 16/24] Take off co2box from the regular EC-EARTH-CC version
 because that mixes up the co2mass preference for TM5 #785 #792.

---
 ece2cmor3/components.py      | 17 +++++++++--------
 ece2cmor3/resources/prefs.py |  6 ++++++
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/ece2cmor3/components.py b/ece2cmor3/components.py
index 1d88e341..a11efc11 100644
--- a/ece2cmor3/components.py
+++ b/ece2cmor3/components.py
@@ -21,14 +21,15 @@
                   table_file: os.path.join(os.path.dirname(__file__), "resources", "co2boxpar.json")}
           }
 
-ece_configs = {'EC-EARTH-AOGCM'   : ["ifs", "nemo"               ],
-               'EC-EARTH-HR'      : ["ifs", "nemo"               ],
-               'EC-EARTH-LR'      : ["ifs", "nemo"               ],
-               'EC-EARTH-CC'      : ["ifs", "nemo", "tm5", "lpjg", "co2box"],
-               'EC-EARTH-GrisIS'  : ["ifs", "nemo"               ], # If a PISM component is added to ece2cmor3 it needs here to be added as well.
-               'EC-EARTH-AerChem' : ["ifs", "nemo", "tm5"        ],
-               'EC-EARTH-Veg'     : ["ifs", "nemo", "lpjg"       ],
-               'EC-EARTH-Veg-LR'  : ["ifs", "nemo", "lpjg"       ]}
+ece_configs = {'EC-EARTH-AOGCM'   : ["ifs", "nemo"                         ],
+               'EC-EARTH-HR'      : ["ifs", "nemo"                         ],
+               'EC-EARTH-LR'      : ["ifs", "nemo"                         ],
+               'EC-EARTH-CC'      : ["ifs", "nemo", "tm5", "lpjg"          ],
+               'EC-EARTH-ESM'     : ["ifs", "nemo",        "lpjg", "co2box"], # If a PISM component is added to ece2cmor3 it needs here to be added as well.
+               'EC-EARTH-GrisIS'  : ["ifs", "nemo"                         ], # If a PISM component is added to ece2cmor3 it needs here to be added as well.
+               'EC-EARTH-AerChem' : ["ifs", "nemo", "tm5"                  ],
+               'EC-EARTH-Veg'     : ["ifs", "nemo",        "lpjg"          ],
+               'EC-EARTH-Veg-LR'  : ["ifs", "nemo",        "lpjg"          ]}
 
 def load_parameter_table(component, filename):
     if component in models:
diff --git a/ece2cmor3/resources/prefs.py b/ece2cmor3/resources/prefs.py
index 3661c4dc..304366f8 100644
--- a/ece2cmor3/resources/prefs.py
+++ b/ece2cmor3/resources/prefs.py
@@ -49,6 +49,12 @@ def keep_variable(target, model_component, ecearth_config):
         # The list above from the second line on is created by using:
         #  more basic-flat-cmip6-file_def_nemo.xml |grep pisces| sed -e 's/field_ref.*//' -e 's/^.*name=//' | sed -e 's/" .*$/",/' |sort |uniq > pisces-vars.txt
 
+    if variable == "co2mass":
+        if ecearth_config == "EC-EARTH-ESM":
+            return model_component == "co2box"
+        else:
+            return model_component == "tm5"
+
     return True
 
 def choose_variable(target_list, model_component, ecearth_config):

From 89c92ca050b903857fac1f1e094c55210da97622 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 16 Feb 2024 10:58:08 +0100
Subject: [PATCH 17/24] Add script which adds the non-cmor HTESSELmon variables
 (the HTESSEL vegetation related variables) #802.

---
 .../add-htessel-vegetation-variables.sh       | 101 ++++++++++++++++++
 1 file changed, 101 insertions(+)
 create mode 100755 ece2cmor3/scripts/add-htessel-vegetation-variables.sh

diff --git a/ece2cmor3/scripts/add-htessel-vegetation-variables.sh b/ece2cmor3/scripts/add-htessel-vegetation-variables.sh
new file mode 100755
index 00000000..ce78d23d
--- /dev/null
+++ b/ece2cmor3/scripts/add-htessel-vegetation-variables.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+# Thomas Reerink
+#
+# This script adds some non-cmor variables (which thus do not exit in
+# the CMIP6 data request) to the Eyr & Emon CMOR tables.
+#
+# This script requires no arguments.
+#
+# Run example:
+#  ./add-htessel-vegetation-variables.sh
+#
+
+if [ "$#" -eq 0 ]; then
+
+ add_the_htessel_vegetation_variables=True
+
+ if [ add_the_htessel_vegetation_variables ]; then
+  # See #802       https://github.com/EC-Earth/ece2cmor3/issues/#802
+  # See #1312-106  https://dev.ec-earth.org/issues/1312#note-106
+
+  # Overview of added variables:
+  # https://codes.ecmwf.int/grib/param-db/27 27.128 Low  vegetation cover (cvl)                HTESSSELmon cvl     Lmon c3PftFrac is taken as basis
+  # https://codes.ecmwf.int/grib/param-db/28 28.128 High vegetation cover (cvh)                HTESSSELmon cvh     Lmon c3PftFrac is taken as basis
+  # https://codes.ecmwf.int/grib/param-db/29 29.128 Type of low vegetation (tvl)               HTESSSELmon tvl     Lmon c3PftFrac is taken as basis
+  # https://codes.ecmwf.int/grib/param-db/30 30.128 Type of high vegetation (tvh)              HTESSSELmon tvh     Lmon c3PftFrac is taken as basis
+  # https://codes.ecmwf.int/grib/param-db/66 66.128 Leaf area index, low vegetation (lai_lv)   HTESSSELmon lai_lv  Lmon lai       is taken as basis
+  # https://codes.ecmwf.int/grib/param-db/67 67.128 Leaf area index, high vegetation (lai_hv)  HTESSSELmon lai_hv  Lmon lai       is taken as basis
+
+  table_path=../resources/cmip6-cmor-tables/Tables/
+  table_file_cv=CMIP6_CV.json
+  table_file_HTESSELmon=CMIP6_HTESSELmon.json
+
+  cd ${table_path}
+  rm -f ${table_file_HTESSELmon}
+  git checkout ${table_file_cv}
+
+  sed -i  '/"IfxAnt"/i \
+            "HTESSELmon",
+  ' ${table_file_cv}
+
+  # Add CMIP6 HTESSELmon table header:
+  echo '{                                              ' | sed 's/\s*$//g' >  ${table_file_HTESSELmon}
+  echo '    "Header": {                                ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "data_specs_version": "01.00.33",      ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "cmor_version": "3.5",                 ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "table_id": "Table HTESSELmon",        ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "realm": "land",                       ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "table_date": "18 November 2020",      ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "missing_value": "1e20",               ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "int_missing_value": "-999",           ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "product": "model-output",             ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "approx_interval": "30.00000",         ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "generic_levels": "",                  ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "mip_era": "CMIP6",                    ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '        "Conventions": "CF-1.7 CMIP-6.2"       ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '    },                                         ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '    "variable_entry": {                        ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+
+  grep -A 17 -e '"c3PftFrac":' CMIP6_Lmon.json | sed -e 's/c3PftFrac/cvl/g' -e 's/area_fraction/cvl/'                               -e 's/"long_name": ".*"/"long_name": "Low vegetation cover"/'              -e 's/"units": ".*"/"units": "0-1"/'    -e 's/"comment": ".*"/"comment": "This parameter is the fraction of the grid box (0-1) that is covered with vegetation that is classified as low (see https:\/\/codes.ecmwf.int\/grib\/param-db\/27)."/g'                  >> ${table_file_HTESSELmon}
+  grep -A 17 -e '"c3PftFrac":' CMIP6_Lmon.json | sed -e 's/c3PftFrac/cvh/g' -e 's/area_fraction/cvh/'                               -e 's/"long_name": ".*"/"long_name": "High vegetation cover"/'             -e 's/"units": ".*"/"units": "0-1"/'    -e 's/"comment": ".*"/"comment": "This parameter is the fraction of the grid box (0-1) that is covered with vegetation that is classified as high (see https:\/\/codes.ecmwf.int\/grib\/param-db\/28)."/g'                 >> ${table_file_HTESSELmon}
+  grep -A 17 -e '"c3PftFrac":' CMIP6_Lmon.json | sed -e 's/c3PftFrac/tvl/g' -e 's/area_fraction/tvl/'                               -e 's/"long_name": ".*"/"long_name": "Type of low vegetation"/'            -e 's/"units": ".*"/"units": "-"/'      -e 's/"comment": ".*"/"comment": "This parameter indicates the 10 types of low vegetation recognised by the ECMWF Integrated Forecasting System (see https:\/\/codes.ecmwf.int\/grib\/param-db\/29)."/g'                   >> ${table_file_HTESSELmon}
+  grep -A 17 -e '"c3PftFrac":' CMIP6_Lmon.json | sed -e 's/c3PftFrac/tvh/g' -e 's/area_fraction/tvh/'                               -e 's/"long_name": ".*"/"long_name": "Type of high vegetation"/'           -e 's/"units": ".*"/"units": "-"/'      -e 's/"comment": ".*"/"comment": "This parameter indicates the 6 types of high vegetation recognised by the ECMWF Integrated Forecasting System (see https:\/\/codes.ecmwf.int\/grib\/param-db\/30)."/g'                   >> ${table_file_HTESSELmon}
+  grep -A 17 -e '"lai":'       CMIP6_Lmon.json | sed -e 's/lai/lai_lv/g'    -e 's/leaf_area_index/leaf_area_index_low_vegetation/'  -e 's/"long_name": ".*"/"long_name": "Leaf area index, low vegetation"/'   -e 's/"units": ".*"/"units": "m2 m-2"/' -e 's/"comment": ".*"/"comment": "This parameter is the surface area of one side of all the leaves found over an area of land for vegetation classified as low (see https:\/\/codes.ecmwf.int\/grib\/param-db\/66)."/g'    >> ${table_file_HTESSELmon}
+  grep -A 16 -e '"lai":'       CMIP6_Lmon.json | sed -e 's/lai/lai_hv/g'    -e 's/leaf_area_index/leaf_area_index_high_vegetation/' -e 's/"long_name": ".*"/"long_name": "Leaf area index, high vegetation"/'  -e 's/"units": ".*"/"units": "m2 m-2"/' -e 's/"comment": ".*"/"comment": "This parameter is the surface area of one side of all the leaves found over an area of land for vegetation classified as high (see https:\/\/codes.ecmwf.int\/grib\/param-db\/67)."/g'   >> ${table_file_HTESSELmon}
+
+  sed -i -e 's/typec3pft/type/' ${table_file_HTESSELmon}
+
+  # Add closing part of CMIP6 table json file:
+  echo '        }                                      ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '    }                                          ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+  echo '}                                              ' | sed 's/\s*$//g' >> ${table_file_HTESSELmon}
+
+
+  # Remove the trailing spaces of the inserted block above:
+  sed -i -e 's/\s*$//g' -e 's/,$/, /g' ${table_file_HTESSELmon}
+  sed -i -e 's/\s*$//g'                ${table_file_cv}
+
+  cd -
+
+  echo
+  echo " $0 reports:"
+  echo "  The adjusted files are:"
+  echo "   ${table_path}/${table_file_cv}"
+  echo "  Added files are:"
+  echo "   ${table_path}/${table_file_HTESSELmon}"
+  echo "  Which is part of a nested repository, therefore to view the diff, run:"
+  echo "  cd ${table_path}; git diff; cd -"
+  echo
+
+ else
+    echo
+    echo " Nothing done, no set of variables and / or experiments has been selected to add to the tables."
+    echo
+ fi
+
+else
+    echo
+    echo " This scripts requires no argument:"
+    echo "  $0"
+    echo
+fi

From 66c0c22640fb034efb2c1b9a52814f06f6344809 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 16 Feb 2024 10:59:58 +0100
Subject: [PATCH 18/24] Add the IFS/HTESSEL vegetation variables to the basic
 admin within the MFPPHY list #802.

---
 ece2cmor3/resources/grib_codes.json | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ece2cmor3/resources/grib_codes.json b/ece2cmor3/resources/grib_codes.json
index ab47dff6..d537a30e 100644
--- a/ece2cmor3/resources/grib_codes.json
+++ b/ece2cmor3/resources/grib_codes.json
@@ -85,6 +85,10 @@
     "152.126",
     "8.128",
     "9.128",
+    "27.128",
+    "28.128",
+    "29.128",
+    "30.128",
     "31.128",
     "32.128",
     "33.128",
@@ -104,6 +108,8 @@
     "50.128",
     "57.128",
     "58.128",
+    "66.128",
+    "67.128",
     "78.128",
     "79.128",
     "121.128",

From 8fda529b2820819937465cf4dc4d9d2a2bc4b2a2 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 16 Feb 2024 11:00:37 +0100
Subject: [PATCH 19/24] Add the IFS/HTESSEL vegetation variables to the ifspar
 #802.

---
 ece2cmor3/resources/ifspar.json | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/ece2cmor3/resources/ifspar.json b/ece2cmor3/resources/ifspar.json
index ad301a40..f423671d 100644
--- a/ece2cmor3/resources/ifspar.json
+++ b/ece2cmor3/resources/ifspar.json
@@ -195,6 +195,36 @@
         "target": "mrros",
         "masked": "land"
     },
+    {
+        "source": "27.128",
+        "target": "cvl",
+        "masked": "land"
+    },
+    {
+        "source": "28.128",
+        "target": "cvh",
+        "masked": "land"
+    },
+    {
+        "source": "29.128",
+        "target": "tvl",
+        "masked": "land"
+    },
+    {
+        "source": "30.128",
+        "target": "tvh",
+        "masked": "land"
+    },
+    {
+        "source": "66.128",
+        "target": "lai_lv",
+        "masked": "land"
+    },
+    {
+        "source": "67.128",
+        "target": "lai_hv",
+        "masked": "land"
+    },
     {
         "source": "9.128",
         "convert": "vol2flux",

From e0cf5d77bda4a178f2dd9816e38f01a78e32b3a5 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 16 Feb 2024 11:02:48 +0100
Subject: [PATCH 20/24] Add the IFS/HTESSEL vegetation variables to the
 OptimESM request and add the table inject script for genecec to include them
 in the optimesm output-control-files #802.

---
 .../optimesm-request-EC-EARTH-CC-varlist.json             | 8 ++++++++
 ece2cmor3/scripts/genecec-for-individual-experiments.sh   | 1 +
 2 files changed, 9 insertions(+)

diff --git a/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json b/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
index f307e725..38c3efe4 100644
--- a/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
+++ b/ece2cmor3/resources/miscellaneous-data-requests/optimesm-request/optimesm-request-EC-EARTH-CC-varlist.json
@@ -69,6 +69,14 @@
         "Emon": [
             "mrsol"
         ],
+        "HTESSELmon": [
+            "cvl",
+            "cvh",
+            "tvl",
+            "tvh",
+            "lai_lv",
+            "lai_hv"
+        ],
         "LImon": [
             "snc",
             "snd",
diff --git a/ece2cmor3/scripts/genecec-for-individual-experiments.sh b/ece2cmor3/scripts/genecec-for-individual-experiments.sh
index 23021bb1..8ab908c4 100755
--- a/ece2cmor3/scripts/genecec-for-individual-experiments.sh
+++ b/ece2cmor3/scripts/genecec-for-individual-experiments.sh
@@ -79,6 +79,7 @@ if [ "$#" -eq 5 ]; then
   # optimesm only:
   if [ ${data_request_file##*/} = 'optimesm-request-EC-EARTH-CC-varlist.json' ]; then
    ./add-lpjg-cc-diagnostics.sh
+   ./add-htessel-vegetation-variables.sh
   fi
 
   rm -rf   ${output_dir}

From 282211ecc80a0e48815cf882c019b7a2593da73e Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 16 Feb 2024 16:08:59 +0100
Subject: [PATCH 21/24] Unify code style with other scripts.

---
 .../scripts/switch-on-off-DynVarMIP-mode.sh   | 31 +++++++++----------
 .../scripts/switch-on-off-pextra-mode.sh      |  2 +-
 2 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/ece2cmor3/scripts/switch-on-off-DynVarMIP-mode.sh b/ece2cmor3/scripts/switch-on-off-DynVarMIP-mode.sh
index 60029bb3..bccdb0f3 100755
--- a/ece2cmor3/scripts/switch-on-off-DynVarMIP-mode.sh
+++ b/ece2cmor3/scripts/switch-on-off-DynVarMIP-mode.sh
@@ -4,12 +4,9 @@
 # This script applies the required changes to run ece2cmor3 & genecec in the DynVarMIP mode or it resets these adjustments.
 # It includes activating the pextra mode.
 #
-# This scripts requires one arguments:
+# This scripts requires one argument: activate-DynVarMIP-mode or deactivate-DynVarMIP-mode
 #
-# ${1} the first   argument is  activate-DynVarMIP-mode  or deactivate-DynVarMIP-mode 
-#
-# Run example:
-#  ./switch-on-off-DynVarMIP-mode.sh activate-DynVarMIP-mode
+# For examples how to call this script, run it without arguments.
 #
 
 if [ "$#" -eq 1 ]; then
@@ -24,19 +21,19 @@ if [ "$#" -eq 1 ]; then
   ./switch-on-off-pextra-mode.sh deactivate-pextra-mode
   git diff ../resources/list-of-ignored-cmip6-requested-variables.xlsx
  else
-    echo '  '
-    echo '  Error: the value of the first argument is wrong.'
-    echo '  '
-    echo '  This scripts requires one argument: There are only two options:'
-    echo '  ' $0 activate-DynVarMIP-mode
-    echo '  ' $0 deactivate-DynVarMIP-mode
-    echo '  '
+  echo
+  echo "  Error: the value of the first argument is wrong."
+  echo
+  echo " This scripts requires one argument: There are only two options:"
+  echo "  $0 activate-DynVarMIP-mode"
+  echo "  $0 deactivate-DynVarMIP-mode"
+  echo
  fi
 
 else
-    echo '  '
-    echo '  This scripts requires one argument: There are only two options:'
-    echo '  ' $0 activate-DynVarMIP-mode
-    echo '  ' $0 deactivate-DynVarMIP-mode
-    echo '  '
+  echo
+  echo " This scripts requires one argument: There are only two options:"
+  echo "  $0 activate-DynVarMIP-mode"
+  echo "  $0 deactivate-DynVarMIP-mode"
+  echo
 fi
diff --git a/ece2cmor3/scripts/switch-on-off-pextra-mode.sh b/ece2cmor3/scripts/switch-on-off-pextra-mode.sh
index 9041c714..5f692efe 100755
--- a/ece2cmor3/scripts/switch-on-off-pextra-mode.sh
+++ b/ece2cmor3/scripts/switch-on-off-pextra-mode.sh
@@ -5,7 +5,7 @@
 #
 # This scripts requires one argument: activate-pextra-mode or deactivate-pextra-mode
 #
-# Run this script without arguments for examples how to call this script.
+# For examples how to call this script, run it without arguments.
 #
 
 if [ "$#" -eq 1 ]; then

From 3ec83d4079edd5db06360988d8248df833824188 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 16 Feb 2024 16:49:16 +0100
Subject: [PATCH 22/24] Unify code style with other scripts.

---
 ece2cmor3/scripts/modify-metadata-template.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ece2cmor3/scripts/modify-metadata-template.sh b/ece2cmor3/scripts/modify-metadata-template.sh
index 8aa9af5a..49867403 100755
--- a/ece2cmor3/scripts/modify-metadata-template.sh
+++ b/ece2cmor3/scripts/modify-metadata-template.sh
@@ -8,7 +8,7 @@
 # third   argument is the ec-earth model configuration
 # fourth  OPTIONAL argument is the meta data template json file which is used as input, the file, e.g. (default): resources/metadata-templates/cmip6-CMIP-piControl-metadata-template.json
 #
-# Run this script without arguments for examples how to call this script.
+# For examples how to call this script, run it without arguments.
 #
 # With this script it is possible to generate a dedicated metadata template json file for each ec-earth3 cmip6 MIP experiment for all active components.
 #
@@ -479,11 +479,11 @@ if [ "$#" -eq 3 ] || [ "$#" -eq 4 ]; then
  echo
 
 else
-    echo '  '
-    echo '  This scripts requires three or four arguments, e.g.:'
-    echo '  ' $0 CMIP historical EC-EARTH-Veg
-    echo '  ' $0 CMIP historical EC-EARTH-CC  ../resources/metadata-templates/cmip6-CMIP-piControl-metadata-template.json
-    echo '  '
+ echo
+ echo "  This scripts requires three or four arguments, e.g.:"
+ echo "   $0 CMIP historical EC-EARTH-Veg"
+ echo "   $0 CMIP historical EC-EARTH-CC  ../resources/metadata-templates/cmip6-CMIP-piControl-metadata-template.json"
+ echo
 fi
 
 

From a8488f8a32bfd5f938e9b90e0e3f4453ee2f7f93 Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 16 Feb 2024 16:50:41 +0100
Subject: [PATCH 23/24] Solve last small issue with clean conflict #802.

---
 .../add-htessel-vegetation-variables.sh       | 36 ++++++++++---------
 .../genecec-for-individual-experiments.sh     |  2 +-
 2 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/ece2cmor3/scripts/add-htessel-vegetation-variables.sh b/ece2cmor3/scripts/add-htessel-vegetation-variables.sh
index ce78d23d..fd432957 100755
--- a/ece2cmor3/scripts/add-htessel-vegetation-variables.sh
+++ b/ece2cmor3/scripts/add-htessel-vegetation-variables.sh
@@ -2,19 +2,18 @@
 # Thomas Reerink
 #
 # This script adds some non-cmor variables (which thus do not exit in
-# the CMIP6 data request) to the Eyr & Emon CMOR tables.
+# the CMIP6 data request) to a new HTESEELmon CMOR table.
 #
-# This script requires no arguments.
+# This script requires one argument.
 #
-# Run example:
-#  ./add-htessel-vegetation-variables.sh
+# For examples how to call this script, run it without arguments.
 #
 
-if [ "$#" -eq 0 ]; then
+if [ "$#" -eq 1 ]; then
 
- add_the_htessel_vegetation_variables=True
+ do_clean=$1
 
- if [ add_the_htessel_vegetation_variables ]; then
+ if [ ${do_clean} == 'clean-before' ] || [ ${do_clean} == 'no-clean-before' ]; then
   # See #802       https://github.com/EC-Earth/ece2cmor3/issues/#802
   # See #1312-106  https://dev.ec-earth.org/issues/1312#note-106
 
@@ -31,8 +30,10 @@ if [ "$#" -eq 0 ]; then
   table_file_HTESSELmon=CMIP6_HTESSELmon.json
 
   cd ${table_path}
-  rm -f ${table_file_HTESSELmon}
-  git checkout ${table_file_cv}
+  if [ ${do_clean} == 'clean-before' ]; then
+   rm -f ${table_file_HTESSELmon}
+   git checkout ${table_file_cv}
+  fi
 
   sed -i  '/"IfxAnt"/i \
             "HTESSELmon",
@@ -88,14 +89,17 @@ if [ "$#" -eq 0 ]; then
   echo
 
  else
-    echo
-    echo " Nothing done, no set of variables and / or experiments has been selected to add to the tables."
-    echo
+  echo
+  echo " This scripts requires one argument: There are only two options:"
+  echo "  $0 clean-before"
+  echo "  $0 no-clean-before"
+  echo
  fi
 
 else
-    echo
-    echo " This scripts requires no argument:"
-    echo "  $0"
-    echo
+  echo
+  echo " This scripts requires one argument: There are only two options:"
+  echo "  $0 clean-before"
+  echo "  $0 no-clean-before"
+  echo
 fi
diff --git a/ece2cmor3/scripts/genecec-for-individual-experiments.sh b/ece2cmor3/scripts/genecec-for-individual-experiments.sh
index 2fb161e0..044c9157 100755
--- a/ece2cmor3/scripts/genecec-for-individual-experiments.sh
+++ b/ece2cmor3/scripts/genecec-for-individual-experiments.sh
@@ -80,7 +80,7 @@ if [ "$#" -eq 5 ]; then
   if [ ${data_request_file##*/} = 'optimesm-request-EC-EARTH-CC-varlist.json' ]; then
    ./add-lpjg-cc-diagnostics.sh
    ./add-variables-for-co2box.sh
-   ./add-htessel-vegetation-variables.sh
+   ./add-htessel-vegetation-variables.sh no-clean-before
   fi
 
   rm -rf   ${output_dir}

From 7da1ad5a8e9589071453601cd0544d1be43eb65e Mon Sep 17 00:00:00 2001
From: treerink <thomas.reerink.knmi@gmail.com>
Date: Fri, 16 Feb 2024 17:14:17 +0100
Subject: [PATCH 24/24] Update header description #778 #794.

---
 ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh b/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
index 5c52c472..e7da2ba0 100755
--- a/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
+++ b/ece2cmor3/scripts/add-lpjg-cc-diagnostics.sh
@@ -2,7 +2,8 @@
 # Thomas Reerink
 #
 # This script adds some non-cmor variables (which thus do not exit in
-# the CMIP6 data request) to the Eyr & Emon CMOR tables.
+# the CMIP6 data request) to the Eyr & Emon CMOR tables. In addition three
+# non-cmor tables (LPJGday, LPJGmon & LPJGyr) and their 18 variables are added.
 #
 # This script requires no arguments.
 #