Update root logger to have the only StreamHandler object

- Update `custom_logger()` to not add StreamHandler and instead inherit from the root logger
- Fix `Run.run_diags()` to properly setup the provenance directory before creating the log file to avoid FileNotFoundError

auxiliary_tools/cdat_regression_testing/909-prov-log/run_script.py

@@ -3,7 +3,8 @@
 SET_NAME = "polar"
 SET_DIR = "909-prov-log"
 
-CFG_PATH: str | None = "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/cdat_regression_testing/909-prov-log/run.cfg"
+# CFG_PATH: str | None = "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/cdat_regression_testing/909-prov-log/run.cfg"
+CFG_PATH = "auxiliary_tools/cdat_regression_testing/909-prov-log/run.cfg"
 MULTIPROCESSING = True
 
 # %%

e3sm_diags/logger.py

@@ -13,7 +13,13 @@
 LOG_LEVEL = logging.INFO
 
 
-# Setup the root logger.
+# Setup the root logger with a default log file.
+# `force` is set to `True` to automatically remove root handlers whenever
+# `basicConfig` called. This is required for cases where multiple e3sm_diags
+# runs are executed. Otherwise, the logger objects attempt to share the same
+# root file reference (which gets deleted between runs), resulting in
+# `FileNotFoundError: [Errno 2] No such file or directory: 'e3sm_diags_run.log'`.
+# More info here: https://stackoverflow.com/a/49202811
 logging.basicConfig(
     format=LOG_FORMAT,
     filename=LOG_FILENAME,
@@ -23,26 +29,20 @@
 )
 logging.captureWarnings(True)
 
-# Capture warnings from other Python packages.
+# Add a console handler to display warnings in the console. This is useful
+# for when other package loggers raise warnings (e.g, NumPy, Xarray).
 console_handler = logging.StreamHandler()
-console_handler.setLevel(logging.WARNING)
+console_handler.setLevel(logging.INFO)
 console_handler.setFormatter(logging.Formatter(LOG_FORMAT))
-logging.getLogger("py.warnings").addHandler(console_handler)
+logging.getLogger().addHandler(console_handler)
 
 
 def custom_logger(name: str, propagate: bool = True) -> logging.Logger:
     """Sets up a custom logger that is a child of the root logger.
 
-    This custom logger inherits the root logger's handlers, but can have its
-    own handlers and settings. This is useful for separating log messages from
-    different parts of the code.
+    This custom logger inherits the root logger's handlers.
+
 
-    `force` is set to `True` to automatically remove root handlers whenever
-    `basicConfig` called. This is required for cases where multiple e3sm_diags
-    runs are executed. Otherwise, the logger objects attempt to share the same
-    root file reference (which gets deleted between runs), resulting in
-    `FileNotFoundError: [Errno 2] No such file or directory: 'e3sm_diags_run.log'`.
-    More info here: https://stackoverflow.com/a/49202811
 
     Parameters
     ----------
@@ -88,17 +88,37 @@ def custom_logger(name: str, propagate: bool = True) -> logging.Logger:
     logger = logging.getLogger(name)
     logger.propagate = propagate
 
-    # This logic prevents duplicate handlers to be added to this logger object,
-    # which avoids repeated log messages.
-    if not logger.handlers:
-        console_handler = logging.StreamHandler()
-        console_handler.setLevel(logging.INFO)
-        console_handler.setFormatter(logging.Formatter(LOG_FORMAT))
-        logger.addHandler(console_handler)
-
     return logger
 
 
+def _update_root_logger_filepath_to_prov_dir(log_path: str):
+    """Updates the log file path to the provenance directory.
+
+    This method changes the log file path to a subdirectory named 'prov'
+    within the given results directory. It updates the filename of the
+    existing file handler to the new path.
+
+    Parameters
+    ----------
+    log_path : str
+        The path to the log file, which is stored in the `results_dir`
+        sub-directory called "prov".
+
+    Notes
+    -----
+    - The method assumes that a logging file handler is already configured.
+    - The log file is closed and reopened at the new location.
+    - The log file mode is determined by the constant `LOG_FILEMODE`.
+    - The log file name is determined by the constant `LOG_FILENAME`.
+    """
+    for handler in logging.root.handlers:
+        if isinstance(handler, logging.FileHandler):
+            handler.baseFilename = log_path
+            handler.stream.close()
+            handler.stream = open(log_path, LOG_FILEMODE)  # type: ignore
+            break
+
+
 def move_log_to_prov_dir(results_dir: str, logger: logging.Logger):
     """Moves the e3sm diags log file to the provenance directory.
 

e3sm_diags/run.py

@@ -1,18 +1,24 @@
 import copy
-import logging
 import os
+import pathlib
 import subprocess
 from datetime import datetime
 from itertools import chain
 from typing import List, Union
 
 import e3sm_diags  # noqa: F401
 from e3sm_diags.e3sm_diags_driver import get_default_diags_path, main
-from e3sm_diags.logger import LOG_FILEMODE, LOG_FILENAME, custom_logger
+from e3sm_diags.logger import (
+    LOG_FILENAME,
+    _update_root_logger_filepath_to_prov_dir,
+    custom_logger,
+)
 from e3sm_diags.parameter import SET_TO_PARAMETERS
 from e3sm_diags.parameter.core_parameter import DEFAULT_SETS, CoreParameter
 from e3sm_diags.parser.core_parser import CoreParser
 
+# Set up a module level logger object. This logger object is a child of the
+# root logger.
 logger = custom_logger(__name__)
 
 
@@ -83,9 +89,13 @@ def run_diags(
         params = self.get_run_parameters(parameters, use_cfg)
         params_results = None
 
-        self.log_path = os.path.join(params[0].results_dir, "prov", LOG_FILENAME)
-        self._update_log_filepath_to_prov_dir()
-        self._log_diagnostic_run_info()
+        # Make the provenance directory to store the log file.
+        prov_dir = os.path.join(params[0].results_dir, "prov")
+        pathlib.Path(prov_dir).mkdir(parents=True, exist_ok=True)
+
+        log_dir = os.path.join(prov_dir, LOG_FILENAME)
+        _update_root_logger_filepath_to_prov_dir(log_dir)
+        self._log_diagnostic_run_info(log_dir)
 
         if params is None or len(params) == 0:
             raise RuntimeError(
@@ -100,15 +110,21 @@ def run_diags(
 
         return params_results
 
-    def _log_diagnostic_run_info(self):
+    def _log_diagnostic_run_info(self, log_path: str):
         """Logs information about the diagnostic run.
 
         This method is useful for tracking the provenance of the diagnostic run
         and understanding the context of the diagnostic results.
 
-        Logs the following information:
-        - Timestamp of the run
-        - Version information (Git branch and commit hash or module version)
+        It logs the following information:
+          - Timestamp of the run
+          - Version information (Git branch and commit hash or module version)
+
+        Parameters
+        ----------
+        log_path : str
+            The path to the log file, which is stored in the `results_dir`
+            sub-directory called "prov".
 
         Notes
         -----
@@ -140,40 +156,14 @@ def _log_diagnostic_run_info(self):
 
         logger.info(
             f"\n{'=' * 80}\n"
-            f"Starting an E3SM Diagnostics run\n"
+            f"E3SM Diagnostics Run\n"
+            f"{'-' * 20}\n"
             f"Timestamp: {timestamp}\n"
             f"Version Info: {version_info}\n"
-            f"Log Filepath: {self.log_path}\n"
+            f"Log Filepath: {log_path}\n"
             f"{'=' * 80}\n"
         )
 
-    def _update_log_filepath_to_prov_dir(self):
-        """Updates the log file path to the provenance directory.
-
-        This method changes the log file path to a subdirectory named 'prov'
-        within the given results directory. It updates the filename of the
-        existing file handler to the new path.
-
-        Parameters
-        ----------
-        results_dir : dir
-            The directory where the results are stored. The log file will be
-            moved to a 'prov' subdirectory within this directory.
-
-        Notes
-        -----
-        - The method assumes that a logging file handler is already configured.
-        - The log file is closed and reopened at the new location.
-        - The log file mode is determined by the constant `LOG_FILEMODE`.
-        - The log file name is determined by the constant `LOG_FILENAME`.
-        """
-        for handler in logging.root.handlers:
-            if isinstance(handler, logging.FileHandler):
-                handler.baseFilename = self.log_path
-                handler.stream.close()
-                handler.stream = open(self.log_path, LOG_FILEMODE)  # type: ignore
-                break
-
     def get_run_parameters(
         self, parameters: List[CoreParameter], use_cfg: bool = True
     ) -> List[CoreParameter]: