diff --git a/auxiliary_tools/cdat_regression_testing/909-prov-log/run_script.py b/auxiliary_tools/cdat_regression_testing/909-prov-log/run_script.py index 6f377440e..46223a19b 100644 --- a/auxiliary_tools/cdat_regression_testing/909-prov-log/run_script.py +++ b/auxiliary_tools/cdat_regression_testing/909-prov-log/run_script.py @@ -3,7 +3,8 @@ SET_NAME = "polar" SET_DIR = "909-prov-log" -CFG_PATH: str | None = "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/cdat_regression_testing/909-prov-log/run.cfg" +# CFG_PATH: str | None = "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/cdat_regression_testing/909-prov-log/run.cfg" +CFG_PATH = "auxiliary_tools/cdat_regression_testing/909-prov-log/run.cfg" MULTIPROCESSING = True # %% diff --git a/e3sm_diags/logger.py b/e3sm_diags/logger.py index a87cb6cab..ae131baf1 100644 --- a/e3sm_diags/logger.py +++ b/e3sm_diags/logger.py @@ -13,7 +13,13 @@ LOG_LEVEL = logging.INFO -# Setup the root logger. +# Setup the root logger with a default log file. +# `force` is set to `True` to automatically remove root handlers whenever +# `basicConfig` called. This is required for cases where multiple e3sm_diags +# runs are executed. Otherwise, the logger objects attempt to share the same +# root file reference (which gets deleted between runs), resulting in +# `FileNotFoundError: [Errno 2] No such file or directory: 'e3sm_diags_run.log'`. +# More info here: https://stackoverflow.com/a/49202811 logging.basicConfig( format=LOG_FORMAT, filename=LOG_FILENAME, @@ -23,26 +29,20 @@ ) logging.captureWarnings(True) -# Capture warnings from other Python packages. +# Add a console handler to display warnings in the console. This is useful +# for when other package loggers raise warnings (e.g, NumPy, Xarray). console_handler = logging.StreamHandler() -console_handler.setLevel(logging.WARNING) +console_handler.setLevel(logging.INFO) console_handler.setFormatter(logging.Formatter(LOG_FORMAT)) -logging.getLogger("py.warnings").addHandler(console_handler) +logging.getLogger().addHandler(console_handler) def custom_logger(name: str, propagate: bool = True) -> logging.Logger: """Sets up a custom logger that is a child of the root logger. - This custom logger inherits the root logger's handlers, but can have its - own handlers and settings. This is useful for separating log messages from - different parts of the code. + This custom logger inherits the root logger's handlers. + - `force` is set to `True` to automatically remove root handlers whenever - `basicConfig` called. This is required for cases where multiple e3sm_diags - runs are executed. Otherwise, the logger objects attempt to share the same - root file reference (which gets deleted between runs), resulting in - `FileNotFoundError: [Errno 2] No such file or directory: 'e3sm_diags_run.log'`. - More info here: https://stackoverflow.com/a/49202811 Parameters ---------- @@ -88,17 +88,37 @@ def custom_logger(name: str, propagate: bool = True) -> logging.Logger: logger = logging.getLogger(name) logger.propagate = propagate - # This logic prevents duplicate handlers to be added to this logger object, - # which avoids repeated log messages. - if not logger.handlers: - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.INFO) - console_handler.setFormatter(logging.Formatter(LOG_FORMAT)) - logger.addHandler(console_handler) - return logger +def _update_root_logger_filepath_to_prov_dir(log_path: str): + """Updates the log file path to the provenance directory. + + This method changes the log file path to a subdirectory named 'prov' + within the given results directory. It updates the filename of the + existing file handler to the new path. + + Parameters + ---------- + log_path : str + The path to the log file, which is stored in the `results_dir` + sub-directory called "prov". + + Notes + ----- + - The method assumes that a logging file handler is already configured. + - The log file is closed and reopened at the new location. + - The log file mode is determined by the constant `LOG_FILEMODE`. + - The log file name is determined by the constant `LOG_FILENAME`. + """ + for handler in logging.root.handlers: + if isinstance(handler, logging.FileHandler): + handler.baseFilename = log_path + handler.stream.close() + handler.stream = open(log_path, LOG_FILEMODE) # type: ignore + break + + def move_log_to_prov_dir(results_dir: str, logger: logging.Logger): """Moves the e3sm diags log file to the provenance directory. diff --git a/e3sm_diags/run.py b/e3sm_diags/run.py index 8503adee3..b9585f214 100644 --- a/e3sm_diags/run.py +++ b/e3sm_diags/run.py @@ -1,6 +1,6 @@ import copy -import logging import os +import pathlib import subprocess from datetime import datetime from itertools import chain @@ -8,11 +8,17 @@ import e3sm_diags # noqa: F401 from e3sm_diags.e3sm_diags_driver import get_default_diags_path, main -from e3sm_diags.logger import LOG_FILEMODE, LOG_FILENAME, custom_logger +from e3sm_diags.logger import ( + LOG_FILENAME, + _update_root_logger_filepath_to_prov_dir, + custom_logger, +) from e3sm_diags.parameter import SET_TO_PARAMETERS from e3sm_diags.parameter.core_parameter import DEFAULT_SETS, CoreParameter from e3sm_diags.parser.core_parser import CoreParser +# Set up a module level logger object. This logger object is a child of the +# root logger. logger = custom_logger(__name__) @@ -83,9 +89,13 @@ def run_diags( params = self.get_run_parameters(parameters, use_cfg) params_results = None - self.log_path = os.path.join(params[0].results_dir, "prov", LOG_FILENAME) - self._update_log_filepath_to_prov_dir() - self._log_diagnostic_run_info() + # Make the provenance directory to store the log file. + prov_dir = os.path.join(params[0].results_dir, "prov") + pathlib.Path(prov_dir).mkdir(parents=True, exist_ok=True) + + log_dir = os.path.join(prov_dir, LOG_FILENAME) + _update_root_logger_filepath_to_prov_dir(log_dir) + self._log_diagnostic_run_info(log_dir) if params is None or len(params) == 0: raise RuntimeError( @@ -100,15 +110,21 @@ def run_diags( return params_results - def _log_diagnostic_run_info(self): + def _log_diagnostic_run_info(self, log_path: str): """Logs information about the diagnostic run. This method is useful for tracking the provenance of the diagnostic run and understanding the context of the diagnostic results. - Logs the following information: - - Timestamp of the run - - Version information (Git branch and commit hash or module version) + It logs the following information: + - Timestamp of the run + - Version information (Git branch and commit hash or module version) + + Parameters + ---------- + log_path : str + The path to the log file, which is stored in the `results_dir` + sub-directory called "prov". Notes ----- @@ -140,40 +156,14 @@ def _log_diagnostic_run_info(self): logger.info( f"\n{'=' * 80}\n" - f"Starting an E3SM Diagnostics run\n" + f"E3SM Diagnostics Run\n" + f"{'-' * 20}\n" f"Timestamp: {timestamp}\n" f"Version Info: {version_info}\n" - f"Log Filepath: {self.log_path}\n" + f"Log Filepath: {log_path}\n" f"{'=' * 80}\n" ) - def _update_log_filepath_to_prov_dir(self): - """Updates the log file path to the provenance directory. - - This method changes the log file path to a subdirectory named 'prov' - within the given results directory. It updates the filename of the - existing file handler to the new path. - - Parameters - ---------- - results_dir : dir - The directory where the results are stored. The log file will be - moved to a 'prov' subdirectory within this directory. - - Notes - ----- - - The method assumes that a logging file handler is already configured. - - The log file is closed and reopened at the new location. - - The log file mode is determined by the constant `LOG_FILEMODE`. - - The log file name is determined by the constant `LOG_FILENAME`. - """ - for handler in logging.root.handlers: - if isinstance(handler, logging.FileHandler): - handler.baseFilename = self.log_path - handler.stream.close() - handler.stream = open(self.log_path, LOG_FILEMODE) # type: ignore - break - def get_run_parameters( self, parameters: List[CoreParameter], use_cfg: bool = True ) -> List[CoreParameter]: