From dc07a9c40a628d1a18ed3ab9d2e2fe0357317858 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 9 Apr 2024 17:53:45 -0500 Subject: [PATCH 01/15] Move CLI logging init into a new dedicated module --- conda_build/build.py | 19 +--- conda_build/cli/logging.py | 79 ++++++++++++++ conda_build/cli/main_build.py | 10 +- conda_build/cli/main_convert.py | 8 +- conda_build/cli/main_debug.py | 8 +- conda_build/cli/main_develop.py | 8 +- conda_build/cli/main_inspect.py | 8 +- conda_build/cli/main_metapackage.py | 8 +- conda_build/cli/main_render.py | 12 ++- conda_build/cli/main_skeleton.py | 8 +- conda_build/config.py | 8 +- conda_build/environ.py | 21 +--- conda_build/index.py | 25 +++-- conda_build/inspect_pkg.py | 6 +- conda_build/jinja_context.py | 4 +- conda_build/metadata.py | 14 +-- conda_build/os_utils/macho.py | 5 +- conda_build/os_utils/pyldd.py | 10 +- conda_build/post.py | 7 +- conda_build/render.py | 4 +- conda_build/source.py | 6 +- conda_build/utils.py | 162 ++++++++++++++++------------ conda_build/variants.py | 7 +- conda_build/windows.py | 5 +- tests/test_source.py | 4 +- tests/test_subpackages.py | 3 +- tests/test_utils.py | 101 ++++++++++------- 27 files changed, 353 insertions(+), 207 deletions(-) create mode 100644 conda_build/cli/logging.py diff --git a/conda_build/build.py b/conda_build/build.py index 5d062f7720..fef7826c01 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -19,6 +19,7 @@ import time import warnings from collections import OrderedDict, deque +from logging import getLogger from os.path import dirname, isdir, isfile, islink, join from pathlib import Path from typing import TYPE_CHECKING @@ -94,6 +95,8 @@ if TYPE_CHECKING: from typing import Any, Iterable +log = getLogger(__name__) + if "bsd" in sys.platform: shell_path = "/bin/sh" elif utils.on_win: @@ -924,7 +927,6 @@ def copy_test_source_files(m, destination): clobber=True, ) except OSError as e: - log = utils.get_logger(__name__) log.warning( f"Failed to copy {f} into test files. Error was: {str(e)}" ) @@ -1304,7 +1306,6 @@ def write_about_json(m): extra = m.get_section("extra") # Add burn-in information to extra if m.config.extra_meta: - log = utils.get_logger(__name__) log.info( "Adding the following extra-meta data to about.json: %s", m.config.extra_meta, @@ -1611,7 +1612,6 @@ def post_process_files(m: MetaData, initial_prefix_files): if not os.path.exists(os.path.join(host_prefix, f)): missing.append(f) if len(missing): - log = utils.get_logger(__name__) log.warning( f"The install/build script(s) for {package_name} deleted the following " f"files (from dependencies) from the prefix:\n{missing}\n" @@ -1683,7 +1683,6 @@ def bundle_conda( new_prefix_files: set[str] = set(), **kw, ): - log = utils.get_logger(__name__) log.info("Packaging %s", metadata.dist()) get_all_replacements(metadata.config) files = output.get("files", []) @@ -2176,7 +2175,6 @@ def _write_activation_text(script_path, m): elif os.path.splitext(script_path)[1].lower() == ".sh": _write_sh_activation_text(fh, m) else: - log = utils.get_logger(__name__) log.warning( f"not adding activation to {script_path} - I don't know how to do so for " "this file type" @@ -2317,7 +2315,6 @@ def build( print(utils.get_skip_message(m)) return default_return - log = utils.get_logger(__name__) host_precs = [] build_precs = [] output_metas = [] @@ -2918,8 +2915,6 @@ def _construct_metadata_for_test_from_package(package, config): # This is still necessary for computing the hash correctly though config.variant = hash_input - log = utils.get_logger(__name__) - # get absolute file location local_pkg_location = os.path.normpath(os.path.abspath(os.path.dirname(package))) @@ -3125,7 +3120,6 @@ def _write_test_run_script( shell_files, trace, ): - log = utils.get_logger(__name__) with open(test_run_script, "w") as tf: tf.write( '{source} "{test_env_script}"\n'.format( @@ -3284,7 +3278,6 @@ def test( :param m: Package's metadata. :type m: Metadata """ - log = utils.get_logger(__name__) # we want to know if we're dealing with package input. If so, we can move the input on success. hash_input = {} @@ -3568,7 +3561,6 @@ def tests_failed( dest = join(broken_dir, os.path.basename(pkg)) if move_broken: - log = utils.get_logger(__name__) try: shutil.move(pkg, dest) log.warning( @@ -3719,7 +3711,6 @@ def build_tree( ) ] ) - log = utils.get_logger(__name__) # downstreams can be a dict, for adding capability for worker labels if hasattr(downstreams, "keys"): downstreams = list(downstreams.keys()) @@ -4059,11 +4050,11 @@ def handle_pypi_upload(wheels, config): try: utils.check_call_env(args + [f]) except: - utils.get_logger(__name__).warning( + log.warning( "wheel upload failed - is twine installed?" " Is this package registered?" ) - utils.get_logger(__name__).warning(f"Wheel file left in {f}") + log.warning(f"Wheel file left in {f}") else: print(f"anaconda_upload is not set. Not uploading wheels: {wheels}") diff --git a/conda_build/cli/logging.py b/conda_build/cli/logging.py new file mode 100644 index 0000000000..665555083e --- /dev/null +++ b/conda_build/cli/logging.py @@ -0,0 +1,79 @@ +# Copyright (C) 2014 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + +import os +import os.path +import sys +from logging import INFO, WARNING, Filter, Formatter, StreamHandler, getLogger +from logging.config import dictConfig +from pathlib import Path +from typing import TYPE_CHECKING + +from conda.base.context import context +from yaml import safe_load + +if TYPE_CHECKING: + from logging import Logger, LogRecord + from typing import Self + + +# https://stackoverflow.com/a/31459386/1170370 +class LessThanFilter(Filter): + def __init__(self, exclusive_maximum: int, name: str = "") -> None: + super().__init__(name) + self.max_level = exclusive_maximum + + def filter(self, record: LogRecord) -> bool: + return record.levelno < self.max_level + + +class GreaterThanFilter(Filter): + def __init__(self, exclusive_minimum: int, name: str = "") -> None: + super().__init__(name) + self.min_level = exclusive_minimum + + def filter(self, record: LogRecord) -> bool: + return record.levelno > self.min_level + + +class DuplicateFilter(Filter): + msgs: set[str] = set() + + def filter(self, record: LogRecord) -> bool: + try: + return record.msg not in self.msgs + finally: + self.msgs.add(record.msg) + + @classmethod + def clear(cls: type[Self]) -> None: + cls.msgs.clear() + + +def init_logging(log: Logger) -> None: + """ + Default initialization of logging for conda-build CLI. + + When using conda-build as a CLI tool (not as a library) we wish to limit logging to + avoid duplication and to otherwise offer some default behavior. + """ + config_file = context.conda_build.get("log_config_file") + if config_file: + config_file = os.path.expandvars(config_file) + dictConfig(safe_load(Path(config_file).expanduser().resolve().read_text())) + + # we don't want propagation in CLI, but we do want it in tests + # this is a pytest limitation: https://github.com/pytest-dev/pytest/issues/3697 + getLogger("conda_build").propagate = "PYTEST_CURRENT_TEST" in os.environ + + if not log.handlers: + log.addHandler(stdout := StreamHandler(sys.stdout)) + stdout.addFilter(LessThanFilter(WARNING)) + stdout.addFilter(DuplicateFilter()) + stdout.setFormatter(Formatter("%(levelname)s: %(message)s")) + + log.addHandler(stderr := StreamHandler(sys.stderr)) + stderr.addFilter(GreaterThanFilter(INFO)) + stderr.addFilter(DuplicateFilter()) + stderr.setFormatter(Formatter("%(levelname)s: %(message)s")) diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index 13e129910d..220616cfc8 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -3,11 +3,11 @@ from __future__ import annotations import argparse -import logging import sys import warnings from glob import glob from itertools import chain +from logging import CRITICAL, getLogger from os.path import abspath, expanduser, expandvars from pathlib import Path from typing import TYPE_CHECKING @@ -36,6 +36,8 @@ from argparse import ArgumentParser, Namespace from typing import Sequence +log = getLogger(__name__) + def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: parser = get_render_parser() @@ -509,7 +511,7 @@ def check_recipe(path_list): def output_action(recipe, config): - with LoggingContext(logging.CRITICAL + 1): + with LoggingContext(CRITICAL + 1): config.verbose = False config.debug = False paths = api.get_output_file_paths(recipe, config=config) @@ -531,6 +533,10 @@ def check_action(recipe, config): def execute(args: Sequence[str] | None = None) -> int: + from .logging import init_logging + + init_logging(log) + _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_convert.py b/conda_build/cli/main_convert.py index d30b725b3d..88de0716a1 100644 --- a/conda_build/cli/main_convert.py +++ b/conda_build/cli/main_convert.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -import logging +from logging import getLogger from os.path import abspath, expanduser from typing import TYPE_CHECKING @@ -14,7 +14,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -logging.basicConfig(level=logging.INFO) +log = getLogger(__name__) epilog = """ @@ -127,6 +127,10 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: + from .logging import init_logging + + init_logging(log) + _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_debug.py b/conda_build/cli/main_debug.py index 731f964217..946f40d843 100644 --- a/conda_build/cli/main_debug.py +++ b/conda_build/cli/main_debug.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -import logging import sys +from logging import getLogger from typing import TYPE_CHECKING from conda.base.context import context @@ -17,7 +17,7 @@ from argparse import ArgumentParser from typing import Sequence -logging.basicConfig(level=logging.INFO) +log = getLogger(__name__) def get_parser() -> ArgumentParser: @@ -94,6 +94,10 @@ def get_parser() -> ArgumentParser: def execute(args: Sequence[str] | None = None) -> int: + from .logging import init_logging + + init_logging(log) + parser = get_parser() parsed = parser.parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_develop.py b/conda_build/cli/main_develop.py index 9b680cbf5a..1685fbd3a4 100644 --- a/conda_build/cli/main_develop.py +++ b/conda_build/cli/main_develop.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -import logging +from logging import getLogger from typing import TYPE_CHECKING from conda.base.context import context @@ -19,7 +19,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -logging.basicConfig(level=logging.INFO) +log = getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: @@ -87,6 +87,10 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: + from .logging import init_logging + + init_logging(log) + _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_inspect.py b/conda_build/cli/main_inspect.py index b1c47c0586..0d679b19a4 100644 --- a/conda_build/cli/main_inspect.py +++ b/conda_build/cli/main_inspect.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -import logging import sys +from logging import getLogger from os.path import expanduser from pprint import pprint from typing import TYPE_CHECKING @@ -22,7 +22,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -logging.basicConfig(level=logging.INFO) +log = getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: @@ -195,6 +195,10 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: + from .logging import init_logging + + init_logging(log) + parser, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_metapackage.py b/conda_build/cli/main_metapackage.py index 91d2edcebb..4873045f4a 100644 --- a/conda_build/cli/main_metapackage.py +++ b/conda_build/cli/main_metapackage.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse -import logging +from logging import getLogger from typing import TYPE_CHECKING from conda.base.context import context @@ -20,7 +20,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -logging.basicConfig(level=logging.INFO) +log = getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: @@ -121,6 +121,10 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: + from .logging import init_logging + + init_logging(log) + _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_render.py b/conda_build/cli/main_render.py index 6e6f2bfa41..1b61a371b5 100644 --- a/conda_build/cli/main_render.py +++ b/conda_build/cli/main_render.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse -import logging +from logging import CRITICAL, INFO, basicConfig, getLogger from pprint import pprint from typing import TYPE_CHECKING @@ -26,7 +26,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -log = logging.getLogger(__name__) +log = getLogger(__name__) # see: https://stackoverflow.com/questions/29986185/python-argparse-dict-arg @@ -201,6 +201,10 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: + from .logging import init_logging + + init_logging(log) + _, parsed = parse_args(args) context.__init__(argparse_args=parsed) @@ -232,14 +236,14 @@ def execute(args: Sequence[str] | None = None) -> int: ) if parsed.output: - with LoggingContext(logging.CRITICAL + 1): + with LoggingContext(CRITICAL + 1): paths = api.get_output_file_paths(metadata_tuples, config=config) print("\n".join(sorted(paths))) if parsed.file: m = metadata_tuples[-1][0] api.output_yaml(m, parsed.file, suppress_outputs=True) else: - logging.basicConfig(level=logging.INFO) + basicConfig(level=INFO) for m, _, _ in metadata_tuples: print("--------------") print("Hash contents:") diff --git a/conda_build/cli/main_skeleton.py b/conda_build/cli/main_skeleton.py index 7013e2ffab..9d42644623 100644 --- a/conda_build/cli/main_skeleton.py +++ b/conda_build/cli/main_skeleton.py @@ -2,11 +2,11 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -import logging import os import pkgutil import sys from importlib import import_module +from logging import getLogger from typing import TYPE_CHECKING from conda.base.context import context @@ -19,7 +19,7 @@ from typing import Sequence thisdir = os.path.dirname(os.path.abspath(__file__)) -logging.basicConfig(level=logging.INFO) +log = getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: @@ -53,6 +53,10 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: + from .logging import init_logging + + init_logging(log) + parser, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/config.py b/conda_build/config.py index 465058701f..2360970dfb 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -14,6 +14,7 @@ import shutil import time from collections import namedtuple +from logging import getLogger from os.path import abspath, expanduser, expandvars, join from typing import TYPE_CHECKING @@ -23,7 +24,6 @@ from .utils import ( get_build_folders, get_conda_operation_locks, - get_logger, on_win, rm_rf, ) @@ -33,6 +33,8 @@ from pathlib import Path from typing import Any +log = getLogger(__name__) + invocation_time = "" @@ -324,7 +326,6 @@ def arch(self): @arch.setter def arch(self, value): - log = get_logger(__name__) log.warning( "Setting build arch. This is only useful when pretending to be on another " "arch, such as for rendering necessary dependencies on a non-native arch. " @@ -340,7 +341,6 @@ def platform(self): @platform.setter def platform(self, value): - log = get_logger(__name__) log.warning( "Setting build platform. This is only useful when " "pretending to be on another platform, such as " @@ -839,7 +839,7 @@ def __exit__(self, e_type, e_value, traceback): and e_type is None and not getattr(self, "keep_old_work") ): - get_logger(__name__).info( + log.info( "--dirty flag and --keep-old-work not specified. " "Removing build/test folder after successful build/test.\n" ) diff --git a/conda_build/environ.py b/conda_build/environ.py index 3113ec7f8a..5fc99c4e9c 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -3,7 +3,6 @@ from __future__ import annotations import contextlib -import logging import multiprocessing import os import platform @@ -14,7 +13,7 @@ from collections import defaultdict from functools import lru_cache from glob import glob -from logging import getLogger +from logging import DEBUG, WARNING, getLogger from os.path import join, normpath from typing import TYPE_CHECKING @@ -125,7 +124,6 @@ def verify_git_repo( git_exe, git_dir, git_url, git_commits_since_tag, debug=False, expected_rev="HEAD" ): env = os.environ.copy() - log = utils.get_logger(__name__) stderr = None if debug else subprocess.DEVNULL @@ -240,7 +238,6 @@ def get_git_info(git_exe, repo, debug): :return: """ d = {} - log = utils.get_logger(__name__) stderr = None if debug else subprocess.DEVNULL @@ -841,15 +838,14 @@ def get_install_actions( global cached_precs global last_index_ts - log = utils.get_logger(__name__) - conda_log_level = logging.WARN + conda_log_level = WARNING specs = list(specs) if specs: specs.extend(context.create_default_packages) if verbose or debug: capture = contextlib.nullcontext if debug: - conda_log_level = logging.DEBUG + conda_log_level = DEBUG else: capture = utils.capture for feature, value in feature_list: @@ -988,18 +984,11 @@ def create_env( """ Create a conda envrionment for the given prefix and specs. """ - if config.debug: - external_logger_context = utils.LoggingContext(logging.DEBUG) - else: - external_logger_context = utils.LoggingContext(logging.WARN) - if os.path.exists(prefix): for entry in glob(os.path.join(prefix, "*")): utils.rm_rf(entry) - with external_logger_context: - log = utils.get_logger(__name__) - + with utils.LoggingContext(DEBUG if config.debug else WARNING): # if os.path.isdir(prefix): # utils.rm_rf(prefix) @@ -1195,7 +1184,7 @@ def get_pkg_dirs_locks(dirs, config): def clean_pkg_cache(dist: str, config: Config) -> None: - with utils.LoggingContext(logging.DEBUG if config.debug else logging.WARN): + with utils.LoggingContext(DEBUG if config.debug else WARNING): locks = get_pkg_dirs_locks((config.bldpkgs_dir, *context.pkgs_dirs), config) with utils.try_acquire_locks(locks, timeout=config.timeout): for pkgs_dir in context.pkgs_dirs: diff --git a/conda_build/index.py b/conda_build/index.py index bcb9c6a9d0..ab2a5d7fa4 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -1,8 +1,7 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause -import logging import os -from functools import partial +from logging import CRITICAL, DEBUG, INFO, WARNING, getLogger from os.path import dirname from conda.base.context import context @@ -12,11 +11,8 @@ from conda_index.index import update_index as _update_index from . import utils -from .utils import ( - get_logger, -) -log = get_logger(__name__) +log = getLogger(__name__) local_index_timestamp = 0 @@ -77,14 +73,12 @@ def get_build_index( loggers = utils.LoggingContext.default_loggers + [__name__] if debug: - log_context = partial(utils.LoggingContext, logging.DEBUG, loggers=loggers) + log_level = DEBUG elif verbose: - log_context = partial(utils.LoggingContext, logging.WARN, loggers=loggers) + log_level = WARNING else: - log_context = partial( - utils.LoggingContext, logging.CRITICAL + 1, loggers=loggers - ) - with log_context(): + log_level = CRITICAL + 1 + with utils.LoggingContext(log_level, loggers=loggers): # this is where we add the "local" channel. It's a little smarter than conda, because # conda does not know about our output_folder when it is not the default setting. if os.path.isdir(output_folder): @@ -162,7 +156,12 @@ def _delegated_update_index( dir_path = parent_path subdirs = [dirname] - log_level = logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING + if debug: + log_level = DEBUG + elif verbose: + log_level = INFO + else: + log_level = WARNING with utils.LoggingContext(log_level): return _update_index( dir_path, diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index 54ec2c6f28..5ef5699376 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -7,6 +7,7 @@ import sys from collections import defaultdict from itertools import groupby +from logging import getLogger from operator import itemgetter from os.path import abspath, basename, dirname, exists, join, normcase from pathlib import Path @@ -31,7 +32,6 @@ from .utils import ( comma_join, ensure_list, - get_logger, on_linux, on_mac, on_win, @@ -41,7 +41,7 @@ if TYPE_CHECKING: from typing import Iterable, Literal -log = get_logger(__name__) +log = getLogger(__name__) def which_package( @@ -263,7 +263,7 @@ def inspect_linkages( if relative: precs = list(which_package(relative, prefix)) if len(precs) > 1: - get_logger(__name__).warning( + log.warning( "Warning: %s comes from multiple packages: %s", path, comma_join(map(str, precs)), diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 307a13ecc9..0443b632e5 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -10,6 +10,7 @@ import time from functools import partial from io import StringIO, TextIOBase +from logging import getLogger from subprocess import CalledProcessError from typing import TYPE_CHECKING from warnings import warn @@ -28,7 +29,6 @@ copy_into, ensure_valid_spec, get_installed_packages, - get_logger, rm_rf, ) from .variants import DEFAULT_COMPILERS @@ -41,7 +41,7 @@ if TYPE_CHECKING: from typing import IO, Any -log = get_logger(__name__) +log = getLogger(__name__) class UndefinedNeverFail(jinja2.Undefined): diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 071036bd0c..198572f763 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -14,6 +14,7 @@ from functools import lru_cache from os.path import isdir, isfile, join from typing import TYPE_CHECKING, NamedTuple, overload +from logging import getLogger import yaml from bs4 import UnicodeDammit @@ -67,6 +68,8 @@ "files of conda recipes)" ) +log = getLogger(__name__) + try: Loader = yaml.CLoader except AttributeError: @@ -207,7 +210,7 @@ def get_selectors(config: Config) -> dict[str, bool]: if not np: np = defaults["numpy"] if config.verbose: - utils.get_logger(__name__).warning( + log.warning( "No numpy version specified in conda_build_config.yaml. " "Falling back to default numpy value of {}".format(defaults["numpy"]) ) @@ -281,7 +284,6 @@ def eval_selector(selector_string, namespace, variants_in_place): except NameError as e: missing_var = parseNameNotFound(e) if variants_in_place: - log = utils.get_logger(__name__) log.debug( "Treating unknown selector '" + missing_var + "' as if it was False." ) @@ -379,7 +381,6 @@ def ensure_valid_fields(meta): def _trim_None_strings(meta_dict): - log = utils.get_logger(__name__) for key, value in meta_dict.items(): if hasattr(value, "keys"): meta_dict[key] = _trim_None_strings(value) @@ -977,7 +978,6 @@ def finalize_outputs_pass( if metadata.skip(): continue try: - log = utils.get_logger(__name__) # We should reparse the top-level recipe to get all of our dependencies fixed up. # we base things on base_metadata because it has the record of the full origin recipe if base_metadata.config.verbose: @@ -1027,7 +1027,6 @@ def finalize_outputs_pass( if not permit_unsatisfiable_variants: raise else: - log = utils.get_logger(__name__) log.warning( "Could not finalize metadata due to missing dependencies: " f"{e.packages}" @@ -1247,7 +1246,6 @@ def parse_again( """ assert not self.final, "modifying metadata after finalization" - log = utils.get_logger(__name__) if kw: log.warning( "using unsupported internal conda-build function `parse_again`. Please use " @@ -1457,7 +1455,6 @@ def get_value(self, name, default=None, autotype=True): # The 'source' section can be written a list, in which case the name # is passed in with an index, e.g. get_value('source/0/git_url') if index is None: - log = utils.get_logger(__name__) log.warning( f"No index specified in get_value('{name}'). Assuming index 0." ) @@ -2222,7 +2219,7 @@ def extract_single_output_text( output = output_matches[output_index] if output_matches else "" except ValueError: if not self.path and self.meta.get("extra", {}).get("parent_recipe"): - utils.get_logger(__name__).warning( + log.warning( f"Didn't match any output in raw metadata. Target value was: {output_name}" ) output = "" @@ -2914,7 +2911,6 @@ def _get_used_vars_output_script(self): find_used_variables_in_batch_script(self.config.variant, script) ) else: - log = utils.get_logger(__name__) log.warning( f"Not detecting used variables in output script {script}; conda-build only knows " "how to search .sh and .bat files right now." diff --git a/conda_build/os_utils/macho.py b/conda_build/os_utils/macho.py index 8e02c8ee86..ec4f091273 100644 --- a/conda_build/os_utils/macho.py +++ b/conda_build/os_utils/macho.py @@ -5,12 +5,15 @@ import stat import sys from itertools import islice +from logging import getLogger from subprocess import PIPE, STDOUT, CalledProcessError, Popen, check_output from .. import utils from ..utils import on_mac from .external import find_preferably_prefixed_executable +log = getLogger(__name__) + NO_EXT = ( ".py", ".pyc", @@ -182,7 +185,6 @@ def find_apple_cctools_executable(name, build_prefix, nofail=False): .splitlines()[0] ) except Exception as e: - log = utils.get_logger(__name__) log.error( f"ERROR :: Found `{tool}` but is is an Apple Xcode stub executable\n" f"and it returned an error:\n{e.output}" @@ -257,7 +259,6 @@ def _chmod(filename, mode): try: os.chmod(filename, mode) except (OSError, utils.PermissionError) as e: - log = utils.get_logger(__name__) log.warning(str(e)) diff --git a/conda_build/os_utils/pyldd.py b/conda_build/os_utils/pyldd.py index ff48d5f891..18c34b474a 100644 --- a/conda_build/os_utils/pyldd.py +++ b/conda_build/os_utils/pyldd.py @@ -4,17 +4,17 @@ import argparse import glob -import logging import os import re import struct import sys from functools import partial +from logging import getLogger from pathlib import Path -from ..utils import ensure_list, get_logger, on_linux, on_mac, on_win +from ..utils import ensure_list, on_linux, on_mac, on_win -logging.basicConfig(level=logging.INFO) +log = getLogger(__name__) ''' @@ -690,7 +690,7 @@ def __init__(self, file): (self.shstrndx,) = struct.unpack(endian + "H", file.read(2)) loc = file.tell() if loc != self.ehsize: - get_logger(__name__).warning(f"file.tell()={loc} != ehsize={self.ehsize}") + log.warning(f"file.tell()={loc} != ehsize={self.ehsize}") def __str__(self): return ( @@ -1088,7 +1088,7 @@ def _inspect_linkages_this(filename, sysroot: str = "", arch="native"): except IncompleteRead: # the file was incomplete, can occur if a package ships a test file # which looks like an ELF file but is not. Orange3 does this. - get_logger(__name__).warning(f"problems inspecting linkages for {filename}") + log.warning(f"problems inspecting linkages for {filename}") return None, [], [] dirname = os.path.dirname(filename) results = cf.get_resolved_shared_libraries(dirname, dirname, sysroot) diff --git a/conda_build/post.py b/conda_build/post.py index 42bf319753..1976ae27a4 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -16,6 +16,7 @@ from fnmatch import fnmatch from fnmatch import translate as fnmatch_translate from functools import partial +from logging import getLogger from os.path import ( basename, dirname, @@ -68,6 +69,8 @@ from .metadata import MetaData +log = getLogger(__name__) + filetypes_for_platform = { "win": (DLLfile, EXEfile), "osx": (machofile,), @@ -1421,7 +1424,6 @@ def check_overlinking_impl( sysroot_files.append(replaced) diffs = set(orig_sysroot_files) - set(sysroot_files) if diffs: - log = utils.get_logger(__name__) log.warning( "Partially parsed some '.tbd' files in sysroot %s, pretending .tbds are their install-names\n" "Adding support to 'conda-build' for parsing these in 'liefldd.py' would be easy and useful:\n" @@ -1630,7 +1632,6 @@ def post_process_shared_lib(m, f, files, host_prefix=None): ) elif codefile == machofile: if m.config.host_platform != "osx": - log = utils.get_logger(__name__) log.warning( "Found Mach-O file but patching is only supported on macOS, skipping: %s", path, @@ -1666,7 +1667,6 @@ def fix_permissions(files, prefix): try: lchmod(path, new_mode) except (OSError, utils.PermissionError) as e: - log = utils.get_logger(__name__) log.warning(str(e)) @@ -1687,7 +1687,6 @@ def check_menuinst_json(files, prefix) -> None: return print("Validating Menu/*.json files") - log = utils.get_logger(__name__, dedupe=False) try: import jsonschema from menuinst.utils import data_path diff --git a/conda_build/render.py b/conda_build/render.py index 0c80df0005..302f0b69be 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -13,6 +13,7 @@ from collections import OrderedDict, defaultdict from contextlib import contextmanager from functools import lru_cache +from logging import getLogger from os.path import ( isabs, isdir, @@ -53,6 +54,8 @@ from .config import Config +log = getLogger(__name__) + def odict_representer(dumper, data): return dumper.represent_dict(data.items()) @@ -738,7 +741,6 @@ def finalize_metadata( if build_unsat or host_unsat: m.final = False - log = utils.get_logger(__name__) log.warning( f"Returning non-final recipe for {m.dist()}; one or more dependencies " "was unsatisfiable:" diff --git a/conda_build/source.py b/conda_build/source.py index 983188dd5a..53fd6bd18a 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -9,6 +9,7 @@ import sys import tempfile import time +from logging import getLogger from os.path import abspath, basename, exists, expanduser, isdir, isfile, join, normpath from pathlib import Path from subprocess import CalledProcessError @@ -32,7 +33,6 @@ copy_into, decompressible_exts, ensure_list, - get_logger, on_win, rm_rf, safe_print_unicode, @@ -42,7 +42,7 @@ if TYPE_CHECKING: from typing import Iterable -log = get_logger(__name__) +log = getLogger(__name__) git_submod_re = re.compile(r"(?:.+)\.(.+)\.(?:.+)\s(.+)") ext_re = re.compile(r"(.*?)(\.(?:tar\.)?[^.]+)$") @@ -636,7 +636,7 @@ def get_repository_info(recipe_path): time.ctime(os.path.getmtime(join(recipe_path, "meta.yaml"))), ) except CalledProcessError: - get_logger(__name__).debug("Failed to checkout source in " + recipe_path) + log.debug("Failed to checkout source in " + recipe_path) return "{}, last modified {}".format( recipe_path, time.ctime(os.path.getmtime(join(recipe_path, "meta.yaml"))) ) diff --git a/conda_build/utils.py b/conda_build/utils.py index 4b5fdcc8d2..5720fbb2a9 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -6,8 +6,6 @@ import fnmatch import hashlib import json -import logging -import logging.config import mmap import os import re @@ -27,6 +25,8 @@ from itertools import filterfalse from json.decoder import JSONDecodeError from locale import getpreferredencoding +from logging import INFO, WARNING, Formatter, StreamHandler, getLogger +from logging.config import dictConfig from os import walk from os.path import ( abspath, @@ -66,6 +66,21 @@ from conda.models.version import VersionOrder from conda.utils import unix_path_to_win +from .cli.logging import DuplicateFilter as _DuplicateFilter +from .cli.logging import GreaterThanFilter as _GreaterThanFilter +from .cli.logging import LessThanFilter as _LessThanFilter +from .conda_interface import ( + PackageRecord, + StringIO, + TemporaryDirectory, + VersionOrder, + cc_conda_build, + download, + unix_path_to_win, + win_path_to_unix, +) +from .conda_interface import rm_rf as _rm_rf +from .deprecations import deprecated from .exceptions import BuildLockError if TYPE_CHECKING: @@ -77,6 +92,8 @@ K = TypeVar("K") V = TypeVar("V") +log = getLogger(__name__) + on_win = sys.platform == "win32" on_mac = sys.platform == "darwin" on_linux = sys.platform == "linux" @@ -256,7 +273,6 @@ def _execute(self, *args, **kwargs): except ImportError as e: psutil = None psutil_exceptions = (OSError, ValueError) - log = get_logger(__name__) log.warning(f"psutil import failed. Error was {e}") log.warning( "only disk usage and time statistics will be available. Install psutil to " @@ -548,7 +564,6 @@ def copy_into( src, dst, timeout=900, symlinks=False, lock=None, locking=True, clobber=False ): """Copy all the files and directories in src to the directory dst""" - log = get_logger(__name__) if symlinks and islink(src): try: os.makedirs(os.path.dirname(dst)) @@ -624,7 +639,6 @@ def move_with_fallback(src, dst): copy_into(src, dst) os.unlink(src) except PermissionError: - log = get_logger(__name__) log.debug( f"Failed to copy/remove path from {src} to {dst} due to permission error" ) @@ -1118,7 +1132,6 @@ def convert_path_for_cygwin_or_msys2(exe, path): .decode(getpreferredencoding()) ) except OSError: - log = get_logger(__name__) log.debug( "cygpath executable not found. Passing native path. This is OK for msys2." ) @@ -1274,7 +1287,6 @@ def expand_globs( # File compared to the globs use / as separator independently of the os glob_files = glob(path, recursive=True) if not glob_files: - log = get_logger(__name__) log.error(f"Glob {path} did not match in root_dir {root_dir}") # https://docs.python.org/3/library/glob.html#glob.glob states that # "whether or not the results are sorted depends on the file system". @@ -1318,7 +1330,7 @@ def find_recipe(path: str) -> str: metas = [m for m in VALID_METAS if os.path.isfile(os.path.join(path, m))] if len(metas) == 1: - get_logger(__name__).warning( + log.warning( "Multiple meta files found. " f"The {metas[0]} file in the base directory ({path}) " "will be used." @@ -1356,7 +1368,7 @@ class LoggingContext: "conda_index.index.convert_cache", ] - def __init__(self, level=logging.WARN, handler=None, close=True, loggers=None): + def __init__(self, level=WARNING, handler=None, close=True, loggers=None): self.level = level self.old_levels = {} self.handler = handler @@ -1370,11 +1382,11 @@ def __init__(self, level=logging.WARN, handler=None, close=True, loggers=None): def __enter__(self): for logger in self.loggers: if isinstance(logger, str): - log = logging.getLogger(logger) + log = getLogger(logger) self.old_levels[logger] = log.level log.setLevel( self.level - if ("install" not in logger or self.level < logging.INFO) + if ("install" not in logger or self.level < INFO) else self.level + 10 ) if self.handler: @@ -1384,7 +1396,7 @@ def __enter__(self): def __exit__(self, et, ev, tb): for logger, level in self.old_levels.items(): - logging.getLogger(logger).setLevel(level) + getLogger(logger).setLevel(level) if self.handler: self.logger.removeHandler(self.handler) if self.handler and self.close: @@ -1588,60 +1600,78 @@ def rm_rf(path: str | os.PathLike) -> None: delete_prefix_from_linked_data(str(path)) -# https://stackoverflow.com/a/31459386/1170370 -class LessThanFilter(logging.Filter): - def __init__(self, exclusive_maximum, name=""): - super().__init__(name) - self.max_level = exclusive_maximum - - def filter(self, record): - # non-zero return means we log this message - return 1 if record.levelno < self.max_level else 0 - - -class GreaterThanFilter(logging.Filter): - def __init__(self, exclusive_minimum, name=""): - super().__init__(name) - self.min_level = exclusive_minimum - - def filter(self, record): - # non-zero return means we log this message - return 1 if record.levelno > self.min_level else 0 - - -# unclutter logs - show messages only once -class DuplicateFilter(logging.Filter): - def __init__(self): - self.msgs = set() - - def filter(self, record): - log = record.msg not in self.msgs - self.msgs.add(record.msg) - return int(log) - +deprecated.constant( + "24.5", + "24.7", + "LessThanFilter", + _LessThanFilter, + addendum="Use `conda_build.cli.logging.LessThanFilter` instead.", +) +deprecated.constant( + "24.5", + "24.7", + "GreaterThanFilter", + _GreaterThanFilter, + addendum="Use `conda_build.cli.logging.GreaterThanFilter` instead.", +) +deprecated.constant( + "24.5", + "24.7", + "DuplicateFilter", + _DuplicateFilter, + addendum="Use `conda_build.cli.logging.DuplicateFilter` instead.", +) -dedupe_filter = DuplicateFilter() -info_debug_stdout_filter = LessThanFilter(logging.WARNING) -warning_error_stderr_filter = GreaterThanFilter(logging.INFO) -level_formatter = logging.Formatter("%(levelname)s: %(message)s") +deprecated.constant( + "24.5", + "24.7", + "dedupe_filter", + _dedupe_filter := _DuplicateFilter(), + addendum="Use `conda_build.cli.logging.DuplicateFilter()` instead.", +) +deprecated.constant( + "24.5", + "24.7", + "info_debug_stdout_filter", + _info_debug_stdout_filter := _LessThanFilter(WARNING), + addendum="Use `conda_build.cli.logging.LessThanFilter(logging.WARNING)` instead.", +) +deprecated.constant( + "24.5", + "24.7", + "warning_error_stderr_filter", + _warning_error_stderr_filter := _GreaterThanFilter(INFO), + addendum="Use `conda_build.cli.logging.GreaterThanFilter(logging.INFO)` instead.", +) +deprecated.constant( + "24.5", + "24.7", + "level_formatter", + _level_formatter := Formatter("%(levelname)s: %(message)s"), +) # set filelock's logger to only show warnings by default -logging.getLogger("filelock").setLevel(logging.WARN) +getLogger("filelock").setLevel(WARNING) # quiet some of conda's less useful output -logging.getLogger("conda.core.linked_data").setLevel(logging.WARN) -logging.getLogger("conda.gateways.disk.delete").setLevel(logging.WARN) -logging.getLogger("conda.gateways.disk.test").setLevel(logging.WARN) +getLogger("conda.core.linked_data").setLevel(WARNING) +getLogger("conda.gateways.disk.delete").setLevel(WARNING) +getLogger("conda.gateways.disk.test").setLevel(WARNING) +@deprecated( + "24.5", + "24.7", + addendum="Use `conda_build.cli.logging.DuplicateFilter.msgs.clear` instead.", +) def reset_deduplicator(): """Most of the time, we want the deduplication. There are some cases (tests especially) where we want to be able to control the duplication.""" - global dedupe_filter - dedupe_filter = DuplicateFilter() + _DuplicateFilter.clear() -def get_logger(name, level=logging.INFO, dedupe=True, add_stdout_stderr_handlers=True): +@deprecated("24.5", "24.7", addendum="Use `conda.cli.logging.init_logging` instead.") +def get_logger(name, level=INFO, dedupe=True, add_stdout_stderr_handlers=True): config_file = None if log_config_file := context.conda_build.get("log_config_file"): config_file = abspath(expanduser(expandvars(log_config_file))) @@ -1650,26 +1680,25 @@ def get_logger(name, level=logging.INFO, dedupe=True, add_stdout_stderr_handlers if config_file: with open(config_file) as f: config_dict = yaml.safe_load(f) - logging.config.dictConfig(config_dict) + dictConfig(config_dict) level = config_dict.get("loggers", {}).get(name, {}).get("level", level) - log = logging.getLogger(name) - if log.level != level: - log.setLevel(level) + log = getLogger(name) + log.setLevel(level) if dedupe: - log.addFilter(dedupe_filter) + log.addFilter(_dedupe_filter) # these are defaults. They can be overridden by configuring a log config yaml file. top_pkg = name.split(".")[0] if top_pkg == "conda_build": # we don't want propagation in CLI, but we do want it in tests # this is a pytest limitation: https://github.com/pytest-dev/pytest/issues/3697 - logging.getLogger(top_pkg).propagate = "PYTEST_CURRENT_TEST" in os.environ + getLogger(top_pkg).propagate = "PYTEST_CURRENT_TEST" in os.environ if add_stdout_stderr_handlers and not log.handlers: - stdout_handler = logging.StreamHandler(sys.stdout) - stderr_handler = logging.StreamHandler(sys.stderr) - stdout_handler.addFilter(info_debug_stdout_filter) - stderr_handler.addFilter(warning_error_stderr_filter) - stderr_handler.setFormatter(level_formatter) + stdout_handler = StreamHandler(sys.stdout) + stderr_handler = StreamHandler(sys.stderr) + stdout_handler.addFilter(_info_debug_stdout_filter) + stderr_handler.addFilter(_warning_error_stderr_filter) + stderr_handler.setFormatter(_level_formatter) stdout_handler.setLevel(level) stderr_handler.setLevel(level) log.addHandler(stdout_handler) @@ -1695,7 +1724,6 @@ def merge_or_update_dict( ): if base == new: return base - log = get_logger(__name__) for key, value in new.items(): if key in base or add_missing_keys: base_value = base.get(key, value) @@ -1890,7 +1918,6 @@ def ensure_valid_spec(spec: str | MatchSpec, warn: bool = False) -> str | MatchS else: if "*" not in spec: if match.group(1) not in ("python", "vc") and warn: - log = get_logger(__name__) log.warning( f"Adding .* to spec '{spec}' to ensure satisfiability. Please " "consider putting {{{{ var_name }}}}.* or some relational " @@ -2093,7 +2120,6 @@ def download_channeldata(channel_url): def shutil_move_more_retrying(src, dest, debug_name): - log = get_logger(__name__) log.info(f"Renaming {debug_name} directory '{src}' to '{dest}'") attempts_left = 5 diff --git a/conda_build/variants.py b/conda_build/variants.py index 3eef82266d..7a4a59d320 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -14,16 +14,19 @@ from itertools import product from pathlib import Path from typing import TYPE_CHECKING +from logging import getLogger import yaml from conda.base.context import context -from .utils import ensure_list, get_logger, islist, on_win, trim_empty_keys +from .utils import ensure_list, islist, on_win, trim_empty_keys from .version import _parse as parse_version if TYPE_CHECKING: from typing import Any, Iterable +log = getLogger(__name__) + DEFAULT_VARIANTS = { "python": f"{sys.version_info.major}.{sys.version_info.minor}", "numpy": { @@ -263,7 +266,6 @@ def _combine_spec_dictionaries( for spec_source, spec in specs.items(): if spec: if log_output: - log = get_logger(__name__) log.info(f"Adding in variants from {spec_source}") for k, v in spec.items(): if not keys or k in keys: @@ -496,7 +498,6 @@ def filter_by_key_value(variants, key, values, source_name): if variant.get(key) is not None and variant.get(key) in values: reduced_variants.append(variant) else: - log = get_logger(__name__) log.debug( f"Filtering variant with key {key} not matching target value(s) " f"({values}) from {source_name}, actual {variant.get(key)}" diff --git a/conda_build/windows.py b/conda_build/windows.py index 8643431a5b..1ee4ba10bf 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause import os import pprint +from logging import getLogger from os.path import dirname, isdir, isfile, join # importing setuptools patches distutils so that it knows how to find VC for python 2.7 @@ -22,12 +23,13 @@ from .utils import ( check_call_env, copy_into, - get_logger, path_prepended, write_bat_activation_text, ) from .variants import get_default_variant, set_language_env_vars +log = getLogger(__name__) + VS_VERSION_STRING = { "8.0": "Visual Studio 8 2005", "9.0": "Visual Studio 9 2008", @@ -101,7 +103,6 @@ def msvc_env_cmd(bits, config, override=None): # TODO: this function will likely break on `win-arm64`. However, unless # there's clear user demand, it's not clear that we should invest the # effort into updating a known deprecated function for a new platform. - log = get_logger(__name__) log.warning( "Using legacy MSVC compiler setup. This will be removed in conda-build 4.0. " "If this recipe does not use a compiler, this message is safe to ignore. " diff --git a/tests/test_source.py b/tests/test_source.py index 1cae2f9997..89f4b627e7 100644 --- a/tests/test_source.py +++ b/tests/test_source.py @@ -9,8 +9,8 @@ from conda.gateways.disk.read import compute_sum from conda_build import source +from conda_build.cli.logging import DuplicateFilter from conda_build.source import download_to_cache -from conda_build.utils import reset_deduplicator from .utils import thisdir @@ -198,5 +198,5 @@ def test_append_hash_to_fn(testing_metadata): testing_metadata.meta["source"] = [ {"folder": "f1", "url": os.path.join(thisdir, "archives", "a.tar.bz2")} ] - reset_deduplicator() + DuplicateFilter.msgs.clear() source.provide(testing_metadata) diff --git a/tests/test_subpackages.py b/tests/test_subpackages.py index 881a4eb4cb..e06aeb01b3 100644 --- a/tests/test_subpackages.py +++ b/tests/test_subpackages.py @@ -13,6 +13,7 @@ from conda_build import api, utils from conda_build.exceptions import BuildScriptException, CondaBuildUserError from conda_build.metadata import MetaDataTuple +from conda_build.cli.logging import DuplicateFilter from conda_build.render import finalize_metadata from .utils import get_valid_recipes, subpackage_dir @@ -273,7 +274,7 @@ def test_subpackage_hash_inputs(testing_config): def test_overlapping_files(testing_config, caplog): recipe_dir = os.path.join(subpackage_dir, "_overlapping_files") - utils.reset_deduplicator() + DuplicateFilter.msgs.clear() outputs = api.build(recipe_dir, config=testing_config) assert len(outputs) == 3 assert sum(int("Exact overlap" in rec.message) for rec in caplog.records) == 1 diff --git a/tests/test_utils.py b/tests/test_utils.py index 98733546b5..2d6cc17d20 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,14 +3,18 @@ import os import subprocess import sys +from logging import DEBUG, StreamHandler, getLogger from pathlib import Path from typing import NamedTuple import filelock import pytest -from pytest import MonkeyPatch +from pytest import CaptureFixture, LogCaptureFixture, MonkeyPatch +from pytest_mock import MockerFixture +from yaml import safe_dump -import conda_build.utils as utils +from conda_build import utils +from conda_build.cli.logging import init_logging from conda_build.exceptions import BuildLockError @@ -155,65 +159,88 @@ def test_filter_files(): @pytest.mark.serial -def test_logger_filtering(caplog, capfd): - import logging +def test_logger_filtering(caplog: LogCaptureFixture, capsys: CaptureFixture) -> None: + log = getLogger(__name__) + init_logging(log) + caplog.set_level(DEBUG) - log = utils.get_logger(__name__, level=logging.DEBUG) log.debug("test debug message") log.info("test info message") log.info("test duplicate message") log.info("test duplicate message") log.warning("test warn message") log.error("test error message") - out, err = capfd.readouterr() + + out, err = capsys.readouterr() assert "test debug message" in out - assert "test info message" in out - assert "test warn message" not in out - assert "test error message" not in out assert "test debug message" not in err + + assert "test info message" in out assert "test info message" not in err + + assert "test warn message" not in out assert "test warn message" in err + + assert "test error message" not in out assert "test error message" in err - assert caplog.text.count("duplicate") == 1 - log.removeHandler(logging.StreamHandler(sys.stdout)) - log.removeHandler(logging.StreamHandler(sys.stderr)) + assert out.count("test duplicate message") == 1 + + # cleanup + log.removeHandler(StreamHandler(sys.stdout)) + log.removeHandler(StreamHandler(sys.stderr)) -def test_logger_config_from_file(testing_workdir, capfd, mocker): + +def test_logger_config_from_file( + testing_workdir, + caplog: LogCaptureFixture, + capsys: CaptureFixture, + mocker: MockerFixture, +) -> None: test_file = os.path.join(testing_workdir, "build_log_config.yaml") - with open(test_file, "w") as f: - f.write( - f""" -version: 1 -formatters: - simple: - format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -handlers: - console: - class: logging.StreamHandler - level: WARN - formatter: simple - stream: ext://sys.stdout -loggers: - {__name__}: - level: WARN - handlers: [console] - propagate: no -root: - level: DEBUG - handlers: [console] -""" + Path(test_file).write_text( + safe_dump( + { + "version": 1, + "formatters": { + "simple": { + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "WARN", + "formatter": "simple", + "stream": "ext://sys.stdout", + } + }, + "loggers": { + __name__: { + "level": "WARN", + "handlers": ["console"], + "propagate": False, + } + }, + "root": {"level": "DEBUG", "handlers": ["console"]}, + } ) + ) + mocker.patch( "conda.base.context.Context.conda_build", new_callable=mocker.PropertyMock, return_value={"log_config_file": test_file}, ) - log = utils.get_logger(__name__) + + log = getLogger(__name__) + init_logging(log) + # default log level is INFO, but our config file should set level to DEBUG log.warning("test message") + # output should have gone to stdout according to config above. - out, err = capfd.readouterr() + out, err = capsys.readouterr() assert "test message" in out # make sure that it is not in stderr - this is testing override of defaults. assert "test message" not in err From a8118dbdae356fa4b82114d9b1d61530de6576e2 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 9 Apr 2024 18:18:45 -0500 Subject: [PATCH 02/15] Remove DuplicateFilter.clear --- conda_build/cli/logging.py | 5 ----- conda_build/utils.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/conda_build/cli/logging.py b/conda_build/cli/logging.py index 665555083e..ba4692d9f2 100644 --- a/conda_build/cli/logging.py +++ b/conda_build/cli/logging.py @@ -15,7 +15,6 @@ if TYPE_CHECKING: from logging import Logger, LogRecord - from typing import Self # https://stackoverflow.com/a/31459386/1170370 @@ -46,10 +45,6 @@ def filter(self, record: LogRecord) -> bool: finally: self.msgs.add(record.msg) - @classmethod - def clear(cls: type[Self]) -> None: - cls.msgs.clear() - def init_logging(log: Logger) -> None: """ diff --git a/conda_build/utils.py b/conda_build/utils.py index 5720fbb2a9..252643fbd5 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -1667,7 +1667,7 @@ def rm_rf(path: str | os.PathLike) -> None: def reset_deduplicator(): """Most of the time, we want the deduplication. There are some cases (tests especially) where we want to be able to control the duplication.""" - _DuplicateFilter.clear() + _DuplicateFilter.msgs.clear() @deprecated("24.5", "24.7", addendum="Use `conda.cli.logging.init_logging` instead.") From 490d28bcb06c189fea18fe5033c18e4660ccae84 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 9 Apr 2024 18:40:49 -0500 Subject: [PATCH 03/15] Configure conda_build root logger --- conda_build/cli/logging.py | 8 +++++--- conda_build/cli/main_build.py | 2 +- conda_build/cli/main_convert.py | 2 +- conda_build/cli/main_debug.py | 2 +- conda_build/cli/main_develop.py | 2 +- conda_build/cli/main_inspect.py | 2 +- conda_build/cli/main_metapackage.py | 2 +- conda_build/cli/main_render.py | 2 +- conda_build/cli/main_skeleton.py | 2 +- tests/test_utils.py | 10 +++++----- 10 files changed, 18 insertions(+), 16 deletions(-) diff --git a/conda_build/cli/logging.py b/conda_build/cli/logging.py index ba4692d9f2..e62237334a 100644 --- a/conda_build/cli/logging.py +++ b/conda_build/cli/logging.py @@ -14,7 +14,7 @@ from yaml import safe_load if TYPE_CHECKING: - from logging import Logger, LogRecord + from logging import LogRecord # https://stackoverflow.com/a/31459386/1170370 @@ -46,7 +46,7 @@ def filter(self, record: LogRecord) -> bool: self.msgs.add(record.msg) -def init_logging(log: Logger) -> None: +def init_logging() -> None: """ Default initialization of logging for conda-build CLI. @@ -58,9 +58,11 @@ def init_logging(log: Logger) -> None: config_file = os.path.expandvars(config_file) dictConfig(safe_load(Path(config_file).expanduser().resolve().read_text())) + log = getLogger("conda_build") + # we don't want propagation in CLI, but we do want it in tests # this is a pytest limitation: https://github.com/pytest-dev/pytest/issues/3697 - getLogger("conda_build").propagate = "PYTEST_CURRENT_TEST" in os.environ + log.propagate = "PYTEST_CURRENT_TEST" in os.environ if not log.handlers: log.addHandler(stdout := StreamHandler(sys.stdout)) diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index 220616cfc8..cfe0761740 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -535,7 +535,7 @@ def check_action(recipe, config): def execute(args: Sequence[str] | None = None) -> int: from .logging import init_logging - init_logging(log) + init_logging() _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_convert.py b/conda_build/cli/main_convert.py index 88de0716a1..fb9370e1d7 100644 --- a/conda_build/cli/main_convert.py +++ b/conda_build/cli/main_convert.py @@ -129,7 +129,7 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: from .logging import init_logging - init_logging(log) + init_logging() _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_debug.py b/conda_build/cli/main_debug.py index 946f40d843..c1bd133c19 100644 --- a/conda_build/cli/main_debug.py +++ b/conda_build/cli/main_debug.py @@ -96,7 +96,7 @@ def get_parser() -> ArgumentParser: def execute(args: Sequence[str] | None = None) -> int: from .logging import init_logging - init_logging(log) + init_logging() parser = get_parser() parsed = parser.parse_args(args) diff --git a/conda_build/cli/main_develop.py b/conda_build/cli/main_develop.py index 1685fbd3a4..06c30f1cc6 100644 --- a/conda_build/cli/main_develop.py +++ b/conda_build/cli/main_develop.py @@ -89,7 +89,7 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: from .logging import init_logging - init_logging(log) + init_logging() _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_inspect.py b/conda_build/cli/main_inspect.py index 0d679b19a4..8e4d8f30c2 100644 --- a/conda_build/cli/main_inspect.py +++ b/conda_build/cli/main_inspect.py @@ -197,7 +197,7 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: from .logging import init_logging - init_logging(log) + init_logging() parser, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_metapackage.py b/conda_build/cli/main_metapackage.py index 4873045f4a..660111bc2e 100644 --- a/conda_build/cli/main_metapackage.py +++ b/conda_build/cli/main_metapackage.py @@ -123,7 +123,7 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: from .logging import init_logging - init_logging(log) + init_logging() _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_render.py b/conda_build/cli/main_render.py index 1b61a371b5..f4283f5663 100644 --- a/conda_build/cli/main_render.py +++ b/conda_build/cli/main_render.py @@ -203,7 +203,7 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: from .logging import init_logging - init_logging(log) + init_logging() _, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/conda_build/cli/main_skeleton.py b/conda_build/cli/main_skeleton.py index 9d42644623..236ff7dd30 100644 --- a/conda_build/cli/main_skeleton.py +++ b/conda_build/cli/main_skeleton.py @@ -55,7 +55,7 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: from .logging import init_logging - init_logging(log) + init_logging() parser, parsed = parse_args(args) context.__init__(argparse_args=parsed) diff --git a/tests/test_utils.py b/tests/test_utils.py index 2d6cc17d20..e3c18f68f2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -160,8 +160,8 @@ def test_filter_files(): @pytest.mark.serial def test_logger_filtering(caplog: LogCaptureFixture, capsys: CaptureFixture) -> None: - log = getLogger(__name__) - init_logging(log) + log = getLogger("conda_build.tests") + init_logging() caplog.set_level(DEBUG) log.debug("test debug message") @@ -216,7 +216,7 @@ def test_logger_config_from_file( } }, "loggers": { - __name__: { + "conda_build": { "level": "WARN", "handlers": ["console"], "propagate": False, @@ -233,8 +233,8 @@ def test_logger_config_from_file( return_value={"log_config_file": test_file}, ) - log = getLogger(__name__) - init_logging(log) + log = getLogger("conda_build.tests") + init_logging() # default log level is INFO, but our config file should set level to DEBUG log.warning("test message") From 13cd79c5879097c228ed244cd842b645e9931627 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Thu, 11 Apr 2024 13:56:18 -0500 Subject: [PATCH 04/15] Add news --- news/5275-logging | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 news/5275-logging diff --git a/news/5275-logging b/news/5275-logging new file mode 100644 index 0000000000..cb6c3a91a8 --- /dev/null +++ b/news/5275-logging @@ -0,0 +1,27 @@ +### Enhancements + +* Only customize logging for conda-build's CLI. Do not mess with logging initialization when using conda-build as a library. (#5275) + +### Bug fixes + +* + +### Deprecations + +* Deprecate `conda_build.utils.reset_deduplicator`. Use `conda_build.cli.logging.DuplicateFilter.msgs.clear` instead. (#5275) +* Deprecate `conda_build.utils.get_logger`. Use `conda.cli.logging.init_logging` instead. (#5275) +* Deprecate `conda_build.utils.LessThanFilter`. Use `conda.cli.logging.LessThanFilter` instead. (#5275) +* Deprecate `conda_build.utils.GreaterThanFilter`. Use `conda.cli.logging.GreaterThanFilter` instead. (#5275) +* Deprecate `conda_build.utils.DuplicateFilter`. Use `conda.cli.logging.DuplicateFilter` instead. (#5275) +* Deprecate `conda_build.utils.dedupe_filter`. Use `conda.cli.logging.DuplicateFilter()` instead. (#5275) +* Deprecate `conda_build.utils.info_debug_stdout_filter`. Use `conda.cli.logging.LessThanFilter(WARNING)` instead. (#5275) +* Deprecate `conda_build.utils.warning_error_stderr_filter`. Use `conda.cli.logging.GreaterThanFilter(INFO)` instead. (#5275) +* Deprecate `conda_build.utils.level_formatter`. Unused. (#5275) + +### Docs + +* + +### Other + +* From d764d304c8d7b9ad405c87fa2a8d58b40819503e Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 17 Apr 2024 00:28:21 -0500 Subject: [PATCH 05/15] Set log level to INFO for test_extra_meta --- tests/test_api_build.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_api_build.py b/tests/test_api_build.py index efba89d75d..5d9e637f54 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -7,7 +7,6 @@ from __future__ import annotations import json -import logging import os import re import subprocess @@ -17,6 +16,7 @@ from collections import OrderedDict from contextlib import nullcontext from glob import glob +from logging import INFO from pathlib import Path from shutil import which from typing import TYPE_CHECKING @@ -91,9 +91,7 @@ def represent_ordereddict(dumper, data): class AnacondaClientArgs: - def __init__( - self, specs, token=None, site=None, log_level=logging.INFO, force=False - ): + def __init__(self, specs, token=None, site=None, log_level=INFO, force=False): from binstar_client.utils import parse_specs self.specs = [parse_specs(specs)] @@ -1880,6 +1878,7 @@ def test_extra_meta(testing_config, caplog): recipe_dir = os.path.join(metadata_dir, "_extra_meta") extra_meta_data = {"foo": "bar"} testing_config.extra_meta = extra_meta_data + caplog.set_level(INFO) outputs = api.build(recipe_dir, config=testing_config) about = json.loads(package_has_file(outputs[0], "info/about.json")) assert "foo" in about["extra"] and about["extra"]["foo"] == "bar" From 98e837aeca6cac0af81a2dbf95c9afed65232b33 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 17 Apr 2024 16:55:42 -0500 Subject: [PATCH 06/15] Adjust logging imports --- conda_build/build.py | 4 +-- conda_build/cli/logging.py | 28 ++++++++++---------- conda_build/cli/main_build.py | 6 ++--- conda_build/cli/main_convert.py | 4 +-- conda_build/cli/main_debug.py | 4 +-- conda_build/cli/main_develop.py | 4 +-- conda_build/cli/main_inspect.py | 4 +-- conda_build/cli/main_metapackage.py | 4 +-- conda_build/cli/main_render.py | 8 +++--- conda_build/cli/main_skeleton.py | 4 +-- conda_build/config.py | 4 +-- conda_build/environ.py | 12 ++++----- conda_build/index.py | 16 ++++++------ conda_build/inspect_pkg.py | 4 +-- conda_build/jinja_context.py | 4 +-- conda_build/metadata.py | 4 +-- conda_build/os_utils/macho.py | 4 +-- conda_build/os_utils/pyldd.py | 4 +-- conda_build/post.py | 4 +-- conda_build/render.py | 4 +-- conda_build/source.py | 4 +-- conda_build/utils.py | 40 ++++++++++++++--------------- conda_build/variants.py | 4 +-- conda_build/windows.py | 4 +-- tests/test_api_build.py | 8 +++--- tests/test_utils.py | 12 ++++----- 26 files changed, 102 insertions(+), 100 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index fef7826c01..8bdc8af5d1 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -8,6 +8,7 @@ import fnmatch import json +import logging import os import random import re @@ -19,7 +20,6 @@ import time import warnings from collections import OrderedDict, deque -from logging import getLogger from os.path import dirname, isdir, isfile, islink, join from pathlib import Path from typing import TYPE_CHECKING @@ -95,7 +95,7 @@ if TYPE_CHECKING: from typing import Any, Iterable -log = getLogger(__name__) +log = logging.getLogger(__name__) if "bsd" in sys.platform: shell_path = "/bin/sh" diff --git a/conda_build/cli/logging.py b/conda_build/cli/logging.py index e62237334a..d05477316d 100644 --- a/conda_build/cli/logging.py +++ b/conda_build/cli/logging.py @@ -2,11 +2,11 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations +import logging +import logging.config import os import os.path import sys -from logging import INFO, WARNING, Filter, Formatter, StreamHandler, getLogger -from logging.config import dictConfig from pathlib import Path from typing import TYPE_CHECKING @@ -18,7 +18,7 @@ # https://stackoverflow.com/a/31459386/1170370 -class LessThanFilter(Filter): +class LessThanFilter(logging.Filter): def __init__(self, exclusive_maximum: int, name: str = "") -> None: super().__init__(name) self.max_level = exclusive_maximum @@ -27,7 +27,7 @@ def filter(self, record: LogRecord) -> bool: return record.levelno < self.max_level -class GreaterThanFilter(Filter): +class GreaterThanFilter(logging.Filter): def __init__(self, exclusive_minimum: int, name: str = "") -> None: super().__init__(name) self.min_level = exclusive_minimum @@ -36,7 +36,7 @@ def filter(self, record: LogRecord) -> bool: return record.levelno > self.min_level -class DuplicateFilter(Filter): +class DuplicateFilter(logging.Filter): msgs: set[str] = set() def filter(self, record: LogRecord) -> bool: @@ -55,22 +55,22 @@ def init_logging() -> None: """ config_file = context.conda_build.get("log_config_file") if config_file: - config_file = os.path.expandvars(config_file) - dictConfig(safe_load(Path(config_file).expanduser().resolve().read_text())) + config_file = Path(os.path.expandvars(config_file)).expanduser().resolve() + logging.config.dictConfig(safe_load(config_file.read_text())) - log = getLogger("conda_build") + log = logging.getLogger("conda_build") # we don't want propagation in CLI, but we do want it in tests # this is a pytest limitation: https://github.com/pytest-dev/pytest/issues/3697 log.propagate = "PYTEST_CURRENT_TEST" in os.environ if not log.handlers: - log.addHandler(stdout := StreamHandler(sys.stdout)) - stdout.addFilter(LessThanFilter(WARNING)) + log.addHandler(stdout := logging.StreamHandler(sys.stdout)) + stdout.addFilter(LessThanFilter(logging.WARNING)) stdout.addFilter(DuplicateFilter()) - stdout.setFormatter(Formatter("%(levelname)s: %(message)s")) + stdout.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) - log.addHandler(stderr := StreamHandler(sys.stderr)) - stderr.addFilter(GreaterThanFilter(INFO)) + log.addHandler(stderr := logging.StreamHandler(sys.stderr)) + stderr.addFilter(GreaterThanFilter(logging.INFO)) stderr.addFilter(DuplicateFilter()) - stderr.setFormatter(Formatter("%(levelname)s: %(message)s")) + stderr.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index cfe0761740..a79a256b83 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -3,11 +3,11 @@ from __future__ import annotations import argparse +import logging import sys import warnings from glob import glob from itertools import chain -from logging import CRITICAL, getLogger from os.path import abspath, expanduser, expandvars from pathlib import Path from typing import TYPE_CHECKING @@ -36,7 +36,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -log = getLogger(__name__) +log = logging.getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: @@ -511,7 +511,7 @@ def check_recipe(path_list): def output_action(recipe, config): - with LoggingContext(CRITICAL + 1): + with LoggingContext(logging.CRITICAL + 1): config.verbose = False config.debug = False paths = api.get_output_file_paths(recipe, config=config) diff --git a/conda_build/cli/main_convert.py b/conda_build/cli/main_convert.py index fb9370e1d7..7b086584a0 100644 --- a/conda_build/cli/main_convert.py +++ b/conda_build/cli/main_convert.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -from logging import getLogger +import logging from os.path import abspath, expanduser from typing import TYPE_CHECKING @@ -14,7 +14,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -log = getLogger(__name__) +log = logging.getLogger(__name__) epilog = """ diff --git a/conda_build/cli/main_debug.py b/conda_build/cli/main_debug.py index c1bd133c19..0eb7fc4388 100644 --- a/conda_build/cli/main_debug.py +++ b/conda_build/cli/main_debug.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations +import logging import sys -from logging import getLogger from typing import TYPE_CHECKING from conda.base.context import context @@ -17,7 +17,7 @@ from argparse import ArgumentParser from typing import Sequence -log = getLogger(__name__) +log = logging.getLogger(__name__) def get_parser() -> ArgumentParser: diff --git a/conda_build/cli/main_develop.py b/conda_build/cli/main_develop.py index 06c30f1cc6..72617c3788 100644 --- a/conda_build/cli/main_develop.py +++ b/conda_build/cli/main_develop.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -from logging import getLogger +import logging from typing import TYPE_CHECKING from conda.base.context import context @@ -19,7 +19,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -log = getLogger(__name__) +log = logging.getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: diff --git a/conda_build/cli/main_inspect.py b/conda_build/cli/main_inspect.py index 8e4d8f30c2..9bb8032eca 100644 --- a/conda_build/cli/main_inspect.py +++ b/conda_build/cli/main_inspect.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations +import logging import sys -from logging import getLogger from os.path import expanduser from pprint import pprint from typing import TYPE_CHECKING @@ -22,7 +22,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -log = getLogger(__name__) +log = logging.getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: diff --git a/conda_build/cli/main_metapackage.py b/conda_build/cli/main_metapackage.py index 660111bc2e..525f30e532 100644 --- a/conda_build/cli/main_metapackage.py +++ b/conda_build/cli/main_metapackage.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse -from logging import getLogger +import logging from typing import TYPE_CHECKING from conda.base.context import context @@ -20,7 +20,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -log = getLogger(__name__) +log = logging.getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: diff --git a/conda_build/cli/main_render.py b/conda_build/cli/main_render.py index f4283f5663..d01579f18b 100644 --- a/conda_build/cli/main_render.py +++ b/conda_build/cli/main_render.py @@ -3,7 +3,7 @@ from __future__ import annotations import argparse -from logging import CRITICAL, INFO, basicConfig, getLogger +import logging from pprint import pprint from typing import TYPE_CHECKING @@ -26,7 +26,7 @@ from argparse import ArgumentParser, Namespace from typing import Sequence -log = getLogger(__name__) +log = logging.getLogger(__name__) # see: https://stackoverflow.com/questions/29986185/python-argparse-dict-arg @@ -236,14 +236,14 @@ def execute(args: Sequence[str] | None = None) -> int: ) if parsed.output: - with LoggingContext(CRITICAL + 1): + with LoggingContext(logging.CRITICAL + 1): paths = api.get_output_file_paths(metadata_tuples, config=config) print("\n".join(sorted(paths))) if parsed.file: m = metadata_tuples[-1][0] api.output_yaml(m, parsed.file, suppress_outputs=True) else: - basicConfig(level=INFO) + logging.basicConfig(level=logging.INFO) for m, _, _ in metadata_tuples: print("--------------") print("Hash contents:") diff --git a/conda_build/cli/main_skeleton.py b/conda_build/cli/main_skeleton.py index 236ff7dd30..405b42124b 100644 --- a/conda_build/cli/main_skeleton.py +++ b/conda_build/cli/main_skeleton.py @@ -2,11 +2,11 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations +import logging import os import pkgutil import sys from importlib import import_module -from logging import getLogger from typing import TYPE_CHECKING from conda.base.context import context @@ -19,7 +19,7 @@ from typing import Sequence thisdir = os.path.dirname(os.path.abspath(__file__)) -log = getLogger(__name__) +log = logging.getLogger(__name__) def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: diff --git a/conda_build/config.py b/conda_build/config.py index 2360970dfb..f021ac361b 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -7,6 +7,7 @@ from __future__ import annotations import copy +import logging import math import os import pickle @@ -14,7 +15,6 @@ import shutil import time from collections import namedtuple -from logging import getLogger from os.path import abspath, expanduser, expandvars, join from typing import TYPE_CHECKING @@ -33,7 +33,7 @@ from pathlib import Path from typing import Any -log = getLogger(__name__) +log = logging.getLogger(__name__) invocation_time = "" diff --git a/conda_build/environ.py b/conda_build/environ.py index 5fc99c4e9c..04a47d9ecb 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -3,6 +3,7 @@ from __future__ import annotations import contextlib +import logging import multiprocessing import os import platform @@ -13,7 +14,6 @@ from collections import defaultdict from functools import lru_cache from glob import glob -from logging import DEBUG, WARNING, getLogger from os.path import join, normpath from typing import TYPE_CHECKING @@ -68,7 +68,7 @@ class InstallActionsType(TypedDict): LINK: list[PackageRecord] -log = getLogger(__name__) +log = logging.getLogger(__name__) # these are things that we provide env vars for more explicitly. This list disables the # pass-through of variant values to env vars for these keys. @@ -838,14 +838,14 @@ def get_install_actions( global cached_precs global last_index_ts - conda_log_level = WARNING + conda_log_level = logging.WARNING specs = list(specs) if specs: specs.extend(context.create_default_packages) if verbose or debug: capture = contextlib.nullcontext if debug: - conda_log_level = DEBUG + conda_log_level = logging.DEBUG else: capture = utils.capture for feature, value in feature_list: @@ -988,7 +988,7 @@ def create_env( for entry in glob(os.path.join(prefix, "*")): utils.rm_rf(entry) - with utils.LoggingContext(DEBUG if config.debug else WARNING): + with utils.LoggingContext(logging.DEBUG if config.debug else logging.WARNING): # if os.path.isdir(prefix): # utils.rm_rf(prefix) @@ -1184,7 +1184,7 @@ def get_pkg_dirs_locks(dirs, config): def clean_pkg_cache(dist: str, config: Config) -> None: - with utils.LoggingContext(DEBUG if config.debug else WARNING): + with utils.LoggingContext(logging.DEBUG if config.debug else logging.WARNING): locks = get_pkg_dirs_locks((config.bldpkgs_dir, *context.pkgs_dirs), config) with utils.try_acquire_locks(locks, timeout=config.timeout): for pkgs_dir in context.pkgs_dirs: diff --git a/conda_build/index.py b/conda_build/index.py index ab2a5d7fa4..286552ae98 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -1,7 +1,7 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause import os -from logging import CRITICAL, DEBUG, INFO, WARNING, getLogger +import logging from os.path import dirname from conda.base.context import context @@ -12,7 +12,7 @@ from . import utils -log = getLogger(__name__) +log = logging.getLogger(__name__) local_index_timestamp = 0 @@ -73,11 +73,11 @@ def get_build_index( loggers = utils.LoggingContext.default_loggers + [__name__] if debug: - log_level = DEBUG + log_level = logging.DEBUG elif verbose: - log_level = WARNING + log_level = logging.WARNING else: - log_level = CRITICAL + 1 + log_level = logging.CRITICAL + 1 with utils.LoggingContext(log_level, loggers=loggers): # this is where we add the "local" channel. It's a little smarter than conda, because # conda does not know about our output_folder when it is not the default setting. @@ -157,11 +157,11 @@ def _delegated_update_index( subdirs = [dirname] if debug: - log_level = DEBUG + log_level = logging.DEBUG elif verbose: - log_level = INFO + log_level = logging.INFO else: - log_level = WARNING + log_level = logging.WARNING with utils.LoggingContext(log_level): return _update_index( dir_path, diff --git a/conda_build/inspect_pkg.py b/conda_build/inspect_pkg.py index 5ef5699376..2746aa56b4 100644 --- a/conda_build/inspect_pkg.py +++ b/conda_build/inspect_pkg.py @@ -3,11 +3,11 @@ from __future__ import annotations import json +import logging import os import sys from collections import defaultdict from itertools import groupby -from logging import getLogger from operator import itemgetter from os.path import abspath, basename, dirname, exists, join, normcase from pathlib import Path @@ -41,7 +41,7 @@ if TYPE_CHECKING: from typing import Iterable, Literal -log = getLogger(__name__) +log = logging.getLogger(__name__) def which_package( diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 0443b632e5..b0faa6ba8e 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -4,13 +4,13 @@ import datetime import json +import logging import os import pathlib import re import time from functools import partial from io import StringIO, TextIOBase -from logging import getLogger from subprocess import CalledProcessError from typing import TYPE_CHECKING from warnings import warn @@ -41,7 +41,7 @@ if TYPE_CHECKING: from typing import IO, Any -log = getLogger(__name__) +log = logging.getLogger(__name__) class UndefinedNeverFail(jinja2.Undefined): diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 198572f763..6085d052d5 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -5,6 +5,7 @@ import copy import hashlib import json +import logging import os import re import sys @@ -14,7 +15,6 @@ from functools import lru_cache from os.path import isdir, isfile, join from typing import TYPE_CHECKING, NamedTuple, overload -from logging import getLogger import yaml from bs4 import UnicodeDammit @@ -68,7 +68,7 @@ "files of conda recipes)" ) -log = getLogger(__name__) +log = logging.getLogger(__name__) try: Loader = yaml.CLoader diff --git a/conda_build/os_utils/macho.py b/conda_build/os_utils/macho.py index ec4f091273..57c99d977b 100644 --- a/conda_build/os_utils/macho.py +++ b/conda_build/os_utils/macho.py @@ -1,18 +1,18 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause +import logging import os import re import stat import sys from itertools import islice -from logging import getLogger from subprocess import PIPE, STDOUT, CalledProcessError, Popen, check_output from .. import utils from ..utils import on_mac from .external import find_preferably_prefixed_executable -log = getLogger(__name__) +log = logging.getLogger(__name__) NO_EXT = ( ".py", diff --git a/conda_build/os_utils/pyldd.py b/conda_build/os_utils/pyldd.py index 18c34b474a..c969b8f60b 100644 --- a/conda_build/os_utils/pyldd.py +++ b/conda_build/os_utils/pyldd.py @@ -4,17 +4,17 @@ import argparse import glob +import logging import os import re import struct import sys from functools import partial -from logging import getLogger from pathlib import Path from ..utils import ensure_list, on_linux, on_mac, on_win -log = getLogger(__name__) +log = logging.getLogger(__name__) ''' diff --git a/conda_build/post.py b/conda_build/post.py index 1976ae27a4..f2a3e48995 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -4,6 +4,7 @@ import json import locale +import logging import os import re import shutil @@ -16,7 +17,6 @@ from fnmatch import fnmatch from fnmatch import translate as fnmatch_translate from functools import partial -from logging import getLogger from os.path import ( basename, dirname, @@ -69,7 +69,7 @@ from .metadata import MetaData -log = getLogger(__name__) +log = logging.getLogger(__name__) filetypes_for_platform = { "win": (DLLfile, EXEfile), diff --git a/conda_build/render.py b/conda_build/render.py index 302f0b69be..ee7036a568 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging import os import random import re @@ -13,7 +14,6 @@ from collections import OrderedDict, defaultdict from contextlib import contextmanager from functools import lru_cache -from logging import getLogger from os.path import ( isabs, isdir, @@ -54,7 +54,7 @@ from .config import Config -log = getLogger(__name__) +log = logging.getLogger(__name__) def odict_representer(dumper, data): diff --git a/conda_build/source.py b/conda_build/source.py index 53fd6bd18a..2c574ce0e9 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -3,13 +3,13 @@ from __future__ import annotations import locale +import logging import os import re import shutil import sys import tempfile import time -from logging import getLogger from os.path import abspath, basename, exists, expanduser, isdir, isfile, join, normpath from pathlib import Path from subprocess import CalledProcessError @@ -42,7 +42,7 @@ if TYPE_CHECKING: from typing import Iterable -log = getLogger(__name__) +log = logging.getLogger(__name__) git_submod_re = re.compile(r"(?:.+)\.(.+)\.(?:.+)\s(.+)") ext_re = re.compile(r"(.*?)(\.(?:tar\.)?[^.]+)$") diff --git a/conda_build/utils.py b/conda_build/utils.py index 252643fbd5..9d4444b528 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -6,6 +6,8 @@ import fnmatch import hashlib import json +import logging +import logging.config import mmap import os import re @@ -25,8 +27,6 @@ from itertools import filterfalse from json.decoder import JSONDecodeError from locale import getpreferredencoding -from logging import INFO, WARNING, Formatter, StreamHandler, getLogger -from logging.config import dictConfig from os import walk from os.path import ( abspath, @@ -92,7 +92,7 @@ K = TypeVar("K") V = TypeVar("V") -log = getLogger(__name__) +log = logging.getLogger(__name__) on_win = sys.platform == "win32" on_mac = sys.platform == "darwin" @@ -1368,7 +1368,7 @@ class LoggingContext: "conda_index.index.convert_cache", ] - def __init__(self, level=WARNING, handler=None, close=True, loggers=None): + def __init__(self, level=logging.WARNING, handler=None, close=True, loggers=None): self.level = level self.old_levels = {} self.handler = handler @@ -1382,11 +1382,11 @@ def __init__(self, level=WARNING, handler=None, close=True, loggers=None): def __enter__(self): for logger in self.loggers: if isinstance(logger, str): - log = getLogger(logger) + log = logging.getLogger(logger) self.old_levels[logger] = log.level log.setLevel( self.level - if ("install" not in logger or self.level < INFO) + if ("install" not in logger or self.level < logging.INFO) else self.level + 10 ) if self.handler: @@ -1396,7 +1396,7 @@ def __enter__(self): def __exit__(self, et, ev, tb): for logger, level in self.old_levels.items(): - getLogger(logger).setLevel(level) + logging.getLogger(logger).setLevel(level) if self.handler: self.logger.removeHandler(self.handler) if self.handler and self.close: @@ -1633,30 +1633,30 @@ def rm_rf(path: str | os.PathLike) -> None: "24.5", "24.7", "info_debug_stdout_filter", - _info_debug_stdout_filter := _LessThanFilter(WARNING), + _info_debug_stdout_filter := _LessThanFilter(logging.WARNING), addendum="Use `conda_build.cli.logging.LessThanFilter(logging.WARNING)` instead.", ) deprecated.constant( "24.5", "24.7", "warning_error_stderr_filter", - _warning_error_stderr_filter := _GreaterThanFilter(INFO), + _warning_error_stderr_filter := _GreaterThanFilter(logging.INFO), addendum="Use `conda_build.cli.logging.GreaterThanFilter(logging.INFO)` instead.", ) deprecated.constant( "24.5", "24.7", "level_formatter", - _level_formatter := Formatter("%(levelname)s: %(message)s"), + _level_formatter := logging.Formatter("%(levelname)s: %(message)s"), ) # set filelock's logger to only show warnings by default -getLogger("filelock").setLevel(WARNING) +logging.getLogger("filelock").setLevel(logging.WARNING) # quiet some of conda's less useful output -getLogger("conda.core.linked_data").setLevel(WARNING) -getLogger("conda.gateways.disk.delete").setLevel(WARNING) -getLogger("conda.gateways.disk.test").setLevel(WARNING) +logging.getLogger("conda.core.linked_data").setLevel(logging.WARNING) +logging.getLogger("conda.gateways.disk.delete").setLevel(logging.WARNING) +logging.getLogger("conda.gateways.disk.test").setLevel(logging.WARNING) @deprecated( @@ -1671,7 +1671,7 @@ def reset_deduplicator(): @deprecated("24.5", "24.7", addendum="Use `conda.cli.logging.init_logging` instead.") -def get_logger(name, level=INFO, dedupe=True, add_stdout_stderr_handlers=True): +def get_logger(name, level=logging.INFO, dedupe=True, add_stdout_stderr_handlers=True): config_file = None if log_config_file := context.conda_build.get("log_config_file"): config_file = abspath(expanduser(expandvars(log_config_file))) @@ -1680,9 +1680,9 @@ def get_logger(name, level=INFO, dedupe=True, add_stdout_stderr_handlers=True): if config_file: with open(config_file) as f: config_dict = yaml.safe_load(f) - dictConfig(config_dict) + logging.config.dictConfig(config_dict) level = config_dict.get("loggers", {}).get(name, {}).get("level", level) - log = getLogger(name) + log = logging.getLogger(name) log.setLevel(level) if dedupe: log.addFilter(_dedupe_filter) @@ -1692,10 +1692,10 @@ def get_logger(name, level=INFO, dedupe=True, add_stdout_stderr_handlers=True): if top_pkg == "conda_build": # we don't want propagation in CLI, but we do want it in tests # this is a pytest limitation: https://github.com/pytest-dev/pytest/issues/3697 - getLogger(top_pkg).propagate = "PYTEST_CURRENT_TEST" in os.environ + logging.getLogger(top_pkg).propagate = "PYTEST_CURRENT_TEST" in os.environ if add_stdout_stderr_handlers and not log.handlers: - stdout_handler = StreamHandler(sys.stdout) - stderr_handler = StreamHandler(sys.stderr) + stdout_handler = logging.StreamHandler(sys.stdout) + stderr_handler = logging.StreamHandler(sys.stderr) stdout_handler.addFilter(_info_debug_stdout_filter) stderr_handler.addFilter(_warning_error_stderr_filter) stderr_handler.setFormatter(_level_formatter) diff --git a/conda_build/variants.py b/conda_build/variants.py index 7a4a59d320..4b2e4ff72e 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -5,6 +5,7 @@ from __future__ import annotations +import logging import os.path import re import sys @@ -14,7 +15,6 @@ from itertools import product from pathlib import Path from typing import TYPE_CHECKING -from logging import getLogger import yaml from conda.base.context import context @@ -25,7 +25,7 @@ if TYPE_CHECKING: from typing import Any, Iterable -log = getLogger(__name__) +log = logging.getLogger(__name__) DEFAULT_VARIANTS = { "python": f"{sys.version_info.major}.{sys.version_info.minor}", diff --git a/conda_build/windows.py b/conda_build/windows.py index 1ee4ba10bf..1b0f21101d 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -1,8 +1,8 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause +import logging import os import pprint -from logging import getLogger from os.path import dirname, isdir, isfile, join # importing setuptools patches distutils so that it knows how to find VC for python 2.7 @@ -28,7 +28,7 @@ ) from .variants import get_default_variant, set_language_env_vars -log = getLogger(__name__) +log = logging.getLogger(__name__) VS_VERSION_STRING = { "8.0": "Visual Studio 8 2005", diff --git a/tests/test_api_build.py b/tests/test_api_build.py index 5d9e637f54..9dce9d3b1f 100644 --- a/tests/test_api_build.py +++ b/tests/test_api_build.py @@ -7,6 +7,7 @@ from __future__ import annotations import json +import logging import os import re import subprocess @@ -16,7 +17,6 @@ from collections import OrderedDict from contextlib import nullcontext from glob import glob -from logging import INFO from pathlib import Path from shutil import which from typing import TYPE_CHECKING @@ -91,7 +91,9 @@ def represent_ordereddict(dumper, data): class AnacondaClientArgs: - def __init__(self, specs, token=None, site=None, log_level=INFO, force=False): + def __init__( + self, specs, token=None, site=None, log_level=logging.INFO, force=False + ): from binstar_client.utils import parse_specs self.specs = [parse_specs(specs)] @@ -1878,7 +1880,7 @@ def test_extra_meta(testing_config, caplog): recipe_dir = os.path.join(metadata_dir, "_extra_meta") extra_meta_data = {"foo": "bar"} testing_config.extra_meta = extra_meta_data - caplog.set_level(INFO) + caplog.set_level(logging.INFO) outputs = api.build(recipe_dir, config=testing_config) about = json.loads(package_has_file(outputs[0], "info/about.json")) assert "foo" in about["extra"] and about["extra"]["foo"] == "bar" diff --git a/tests/test_utils.py b/tests/test_utils.py index e3c18f68f2..ea0554b656 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,9 +1,9 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause +import logging import os import subprocess import sys -from logging import DEBUG, StreamHandler, getLogger from pathlib import Path from typing import NamedTuple @@ -160,9 +160,9 @@ def test_filter_files(): @pytest.mark.serial def test_logger_filtering(caplog: LogCaptureFixture, capsys: CaptureFixture) -> None: - log = getLogger("conda_build.tests") + log = logging.getLogger("conda_build.tests") init_logging() - caplog.set_level(DEBUG) + caplog.set_level(logging.DEBUG) log.debug("test debug message") log.info("test info message") @@ -187,8 +187,8 @@ def test_logger_filtering(caplog: LogCaptureFixture, capsys: CaptureFixture) -> assert out.count("test duplicate message") == 1 # cleanup - log.removeHandler(StreamHandler(sys.stdout)) - log.removeHandler(StreamHandler(sys.stderr)) + log.removeHandler(logging.StreamHandler(sys.stdout)) + log.removeHandler(logging.StreamHandler(sys.stderr)) def test_logger_config_from_file( @@ -233,7 +233,7 @@ def test_logger_config_from_file( return_value={"log_config_file": test_file}, ) - log = getLogger("conda_build.tests") + log = logging.getLogger("conda_build.tests") init_logging() # default log level is INFO, but our config file should set level to DEBUG From 2764e0c9eb4114a1c10f16b46cc0d98b90f849e9 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 17 Apr 2024 16:57:53 -0500 Subject: [PATCH 07/15] Move init_logging import to top --- conda_build/cli/main_build.py | 3 +-- conda_build/cli/main_convert.py | 4 ++-- conda_build/cli/main_debug.py | 3 +-- conda_build/cli/main_develop.py | 4 ++-- conda_build/cli/main_inspect.py | 4 ++-- conda_build/cli/main_metapackage.py | 4 ++-- conda_build/cli/main_render.py | 3 +-- conda_build/cli/main_skeleton.py | 3 +-- 8 files changed, 12 insertions(+), 16 deletions(-) diff --git a/conda_build/cli/main_build.py b/conda_build/cli/main_build.py index a79a256b83..c7a6a3ee0c 100644 --- a/conda_build/cli/main_build.py +++ b/conda_build/cli/main_build.py @@ -24,6 +24,7 @@ ) from ..utils import LoggingContext from .actions import KeyValueAction +from .logging import init_logging from .main_render import get_render_parser try: @@ -533,8 +534,6 @@ def check_action(recipe, config): def execute(args: Sequence[str] | None = None) -> int: - from .logging import init_logging - init_logging() _, parsed = parse_args(args) diff --git a/conda_build/cli/main_convert.py b/conda_build/cli/main_convert.py index 7b086584a0..1f4a20b491 100644 --- a/conda_build/cli/main_convert.py +++ b/conda_build/cli/main_convert.py @@ -9,6 +9,8 @@ from conda.base.context import context from .. import api +from ..conda_interface import ArgumentParser +from .logging import init_logging if TYPE_CHECKING: from argparse import ArgumentParser, Namespace @@ -127,8 +129,6 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: - from .logging import init_logging - init_logging() _, parsed = parse_args(args) diff --git a/conda_build/cli/main_debug.py b/conda_build/cli/main_debug.py index 0eb7fc4388..a46b6a30ae 100644 --- a/conda_build/cli/main_debug.py +++ b/conda_build/cli/main_debug.py @@ -11,6 +11,7 @@ from .. import api from ..utils import on_win from . import validators as valid +from .logging import init_logging from .main_render import get_render_parser if TYPE_CHECKING: @@ -94,8 +95,6 @@ def get_parser() -> ArgumentParser: def execute(args: Sequence[str] | None = None) -> int: - from .logging import init_logging - init_logging() parser = get_parser() diff --git a/conda_build/cli/main_develop.py b/conda_build/cli/main_develop.py index 72617c3788..aff7c68bd8 100644 --- a/conda_build/cli/main_develop.py +++ b/conda_build/cli/main_develop.py @@ -8,6 +8,8 @@ from conda.base.context import context from .. import api +from ..conda_interface import ArgumentParser +from .logging import init_logging try: from conda.cli.helpers import add_parser_prefix @@ -87,8 +89,6 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: - from .logging import init_logging - init_logging() _, parsed = parse_args(args) diff --git a/conda_build/cli/main_inspect.py b/conda_build/cli/main_inspect.py index 9bb8032eca..92c34da2c4 100644 --- a/conda_build/cli/main_inspect.py +++ b/conda_build/cli/main_inspect.py @@ -11,6 +11,8 @@ from conda.base.context import context from .. import api +from ..conda_interface import ArgumentParser +from .logging import init_logging try: from conda.cli.helpers import add_parser_prefix @@ -195,8 +197,6 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: - from .logging import init_logging - init_logging() parser, parsed = parse_args(args) diff --git a/conda_build/cli/main_metapackage.py b/conda_build/cli/main_metapackage.py index 525f30e532..da1c56460c 100644 --- a/conda_build/cli/main_metapackage.py +++ b/conda_build/cli/main_metapackage.py @@ -9,6 +9,8 @@ from conda.base.context import context from .. import api +from ..conda_interface import ArgumentParser +from .logging import init_logging try: from conda.cli.helpers import add_parser_channels @@ -121,8 +123,6 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: - from .logging import init_logging - init_logging() _, parsed = parse_args(args) diff --git a/conda_build/cli/main_render.py b/conda_build/cli/main_render.py index d01579f18b..0c2416eaee 100644 --- a/conda_build/cli/main_render.py +++ b/conda_build/cli/main_render.py @@ -15,6 +15,7 @@ from ..config import get_channel_urls, get_or_merge_config from ..utils import LoggingContext from ..variants import get_package_variants, set_language_env_vars +from .logging import init_logging try: from conda.cli.helpers import add_parser_channels @@ -201,8 +202,6 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: - from .logging import init_logging - init_logging() _, parsed = parse_args(args) diff --git a/conda_build/cli/main_skeleton.py b/conda_build/cli/main_skeleton.py index 405b42124b..3d3b0eab09 100644 --- a/conda_build/cli/main_skeleton.py +++ b/conda_build/cli/main_skeleton.py @@ -13,6 +13,7 @@ from .. import api from ..config import Config +from .logging import init_logging if TYPE_CHECKING: from argparse import ArgumentParser, Namespace @@ -53,8 +54,6 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]: def execute(args: Sequence[str] | None = None) -> int: - from .logging import init_logging - init_logging() parser, parsed = parse_args(args) From f93d11c71cf4c30dc989338276d3050207f1153e Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 10 May 2024 17:34:59 -0500 Subject: [PATCH 08/15] Update environ.py --- conda_build/environ.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 04a47d9ecb..ac8735307b 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -import contextlib import logging import multiprocessing import os @@ -12,6 +11,7 @@ import sys import warnings from collections import defaultdict +from contextlib import nullcontext from functools import lru_cache from glob import glob from os.path import join, normpath @@ -838,16 +838,9 @@ def get_install_actions( global cached_precs global last_index_ts - conda_log_level = logging.WARNING specs = list(specs) if specs: specs.extend(context.create_default_packages) - if verbose or debug: - capture = contextlib.nullcontext - if debug: - conda_log_level = logging.DEBUG - else: - capture = utils.capture for feature, value in feature_list: if value: specs.append(f"{feature}@") @@ -881,8 +874,8 @@ def get_install_actions( # this is hiding output like: # Fetching package metadata ........... # Solving package specifications: .......... - with utils.LoggingContext(conda_log_level): - with capture(): + with utils.LoggingContext(logging.DEBUG if debug else logging.WARNING): + with nullcontext() if verbose or debug else utils.capture(): try: _actions = _install_actions(prefix, index, specs, subdir=subdir) precs = _actions["LINK"] From d864abc0a3c971354ce877528f230762e565aa98 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 10 May 2024 18:48:08 -0500 Subject: [PATCH 09/15] Undo conda modifying root logger --- conda_build/cli/logging.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda_build/cli/logging.py b/conda_build/cli/logging.py index d05477316d..109f3117e0 100644 --- a/conda_build/cli/logging.py +++ b/conda_build/cli/logging.py @@ -53,6 +53,9 @@ def init_logging() -> None: When using conda-build as a CLI tool (not as a library) we wish to limit logging to avoid duplication and to otherwise offer some default behavior. """ + # undo conda messing with the root logger + logging.getLogger(None).setLevel(logging.WARNING) + config_file = context.conda_build.get("log_config_file") if config_file: config_file = Path(os.path.expandvars(config_file)).expanduser().resolve() From 9f64bb39dd8420b0d1b7588fe8ba924f57e28425 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 10 May 2024 23:04:36 -0500 Subject: [PATCH 10/15] Restore default log level to INFO --- conda_build/cli/logging.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda_build/cli/logging.py b/conda_build/cli/logging.py index 109f3117e0..6eecbae96f 100644 --- a/conda_build/cli/logging.py +++ b/conda_build/cli/logging.py @@ -63,6 +63,12 @@ def init_logging() -> None: log = logging.getLogger("conda_build") + # historically conda_build has defaulted the logging to INFO and so all of the + # log.info is viewed as default output, until we convert all of the existing + # log.info to standard print statements we will need to continue defaulting to INFO + if log.level == logging.NOTSET: + log.setLevel(logging.INFO) + # we don't want propagation in CLI, but we do want it in tests # this is a pytest limitation: https://github.com/pytest-dev/pytest/issues/3697 log.propagate = "PYTEST_CURRENT_TEST" in os.environ From 27e7a20c9931320894297d3728aa3ac47649f1d8 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Mon, 13 May 2024 16:43:06 -0500 Subject: [PATCH 11/15] Remove unnecessary logging config --- conda_build/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/conda_build/utils.py b/conda_build/utils.py index 9d4444b528..6b77496e67 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -1650,14 +1650,6 @@ def rm_rf(path: str | os.PathLike) -> None: _level_formatter := logging.Formatter("%(levelname)s: %(message)s"), ) -# set filelock's logger to only show warnings by default -logging.getLogger("filelock").setLevel(logging.WARNING) - -# quiet some of conda's less useful output -logging.getLogger("conda.core.linked_data").setLevel(logging.WARNING) -logging.getLogger("conda.gateways.disk.delete").setLevel(logging.WARNING) -logging.getLogger("conda.gateways.disk.test").setLevel(logging.WARNING) - @deprecated( "24.5", From 267b56966c6bf1215eac5f853849e8df76b078db Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 14 May 2024 10:17:01 -0500 Subject: [PATCH 12/15] Correct logging test --- conda_build/cli/logging.py | 15 ++++++++++++--- tests/test_utils.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/conda_build/cli/logging.py b/conda_build/cli/logging.py index 6eecbae96f..d850b6e82d 100644 --- a/conda_build/cli/logging.py +++ b/conda_build/cli/logging.py @@ -7,6 +7,7 @@ import os import os.path import sys +from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING @@ -46,16 +47,20 @@ def filter(self, record: LogRecord) -> bool: self.msgs.add(record.msg) +@lru_cache def init_logging() -> None: """ Default initialization of logging for conda-build CLI. When using conda-build as a CLI tool (not as a library) we wish to limit logging to avoid duplication and to otherwise offer some default behavior. + + This is a onetime initialization that should be called at the start of CLI execution. """ # undo conda messing with the root logger logging.getLogger(None).setLevel(logging.WARNING) + # load the logging configuration from the config file config_file = context.conda_build.get("log_config_file") if config_file: config_file = Path(os.path.expandvars(config_file)).expanduser().resolve() @@ -69,17 +74,21 @@ def init_logging() -> None: if log.level == logging.NOTSET: log.setLevel(logging.INFO) - # we don't want propagation in CLI, but we do want it in tests + # we don't want propagation to the root logger in CLI, but we do want it in tests # this is a pytest limitation: https://github.com/pytest-dev/pytest/issues/3697 log.propagate = "PYTEST_CURRENT_TEST" in os.environ if not log.handlers: + # only add our handlers when none are added via logging.config + + # filter DEBUG/INFO messages to stdout log.addHandler(stdout := logging.StreamHandler(sys.stdout)) stdout.addFilter(LessThanFilter(logging.WARNING)) - stdout.addFilter(DuplicateFilter()) + stdout.addFilter(DuplicateFilter()) # avoid duplicate messages stdout.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + # filter WARNING/ERROR/CRITICAL messages to stderr log.addHandler(stderr := logging.StreamHandler(sys.stderr)) stderr.addFilter(GreaterThanFilter(logging.INFO)) - stderr.addFilter(DuplicateFilter()) + stderr.addFilter(DuplicateFilter()) # avoid duplicate messages stderr.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) diff --git a/tests/test_utils.py b/tests/test_utils.py index ea0554b656..3041826d1d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,22 +1,26 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + import logging import os import subprocess import sys from pathlib import Path -from typing import NamedTuple +from typing import TYPE_CHECKING, NamedTuple import filelock import pytest -from pytest import CaptureFixture, LogCaptureFixture, MonkeyPatch -from pytest_mock import MockerFixture from yaml import safe_dump from conda_build import utils from conda_build.cli.logging import init_logging from conda_build.exceptions import BuildLockError +if TYPE_CHECKING: + from pytest import CaptureFixture, LogCaptureFixture, MonkeyPatch + from pytest_mock import MockerFixture + @pytest.mark.skipif( utils.on_win, reason="only unix has python version in site-packages path" @@ -158,11 +162,13 @@ def test_filter_files(): assert len(utils.filter_files(files_list, "")) == len(files_list) -@pytest.mark.serial def test_logger_filtering(caplog: LogCaptureFixture, capsys: CaptureFixture) -> None: - log = logging.getLogger("conda_build.tests") + log = logging.getLogger("conda_build.test_logger_filtering") init_logging() + + # temporarily override the default log levels so we can test the filtering caplog.set_level(logging.DEBUG) + caplog.set_level(logging.DEBUG, logger="conda_build") log.debug("test debug message") log.info("test info message") @@ -170,6 +176,7 @@ def test_logger_filtering(caplog: LogCaptureFixture, capsys: CaptureFixture) -> log.info("test duplicate message") log.warning("test warn message") log.error("test error message") + log.critical("test critical message") out, err = capsys.readouterr() assert "test debug message" in out @@ -184,11 +191,19 @@ def test_logger_filtering(caplog: LogCaptureFixture, capsys: CaptureFixture) -> assert "test error message" not in out assert "test error message" in err + assert "test critical message" not in out + assert "test critical message" in err + assert out.count("test duplicate message") == 1 + # the duplicate filter is on the conda_build logger, however in testing we + # propagate to the root logger so the root logger will still get the duplicate + # messages + assert caplog.text.count("test duplicate message") == 2 + # cleanup - log.removeHandler(logging.StreamHandler(sys.stdout)) - log.removeHandler(logging.StreamHandler(sys.stderr)) + init_logging.cache_clear() + logging.getLogger("conda_build").handlers.clear() def test_logger_config_from_file( @@ -233,7 +248,7 @@ def test_logger_config_from_file( return_value={"log_config_file": test_file}, ) - log = logging.getLogger("conda_build.tests") + log = logging.getLogger("conda_build.test_logger_config_from_file") init_logging() # default log level is INFO, but our config file should set level to DEBUG @@ -245,6 +260,10 @@ def test_logger_config_from_file( # make sure that it is not in stderr - this is testing override of defaults. assert "test message" not in err + # cleanup + init_logging.cache_clear() + logging.getLogger("conda_build").handlers.clear() + def test_ensure_valid_spec(): assert utils.ensure_valid_spec("python") == "python" From d6fdde429274bc79a04740f90215968528ddacad Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Tue, 14 May 2024 11:40:29 -0500 Subject: [PATCH 13/15] Import warnings, not warn --- conda_build/jinja_context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index b0faa6ba8e..2bc5217385 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -9,11 +9,11 @@ import pathlib import re import time +import warnings from functools import partial from io import StringIO, TextIOBase from subprocess import CalledProcessError from typing import TYPE_CHECKING -from warnings import warn import jinja2 import yaml @@ -213,7 +213,7 @@ def load_setuptools( recipe_dir=None, permit_undefined_jinja=True, ): - warn( + warnings.warn( "conda_build.jinja_context.load_setuptools is pending deprecation in a future release. " "Use conda_build.jinja_context.load_setup_py_data instead.", PendingDeprecationWarning, From fdde7c8984afaa892e81168242b60c2b99658921 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 4 Sep 2024 16:35:22 -0500 Subject: [PATCH 14/15] Correct imports --- conda_build/cli/main_convert.py | 1 - conda_build/cli/main_develop.py | 1 - conda_build/cli/main_inspect.py | 1 - conda_build/cli/main_metapackage.py | 1 - conda_build/utils.py | 11 ----------- 5 files changed, 15 deletions(-) diff --git a/conda_build/cli/main_convert.py b/conda_build/cli/main_convert.py index 1f4a20b491..b88e6ec832 100644 --- a/conda_build/cli/main_convert.py +++ b/conda_build/cli/main_convert.py @@ -9,7 +9,6 @@ from conda.base.context import context from .. import api -from ..conda_interface import ArgumentParser from .logging import init_logging if TYPE_CHECKING: diff --git a/conda_build/cli/main_develop.py b/conda_build/cli/main_develop.py index aff7c68bd8..884f4a90ea 100644 --- a/conda_build/cli/main_develop.py +++ b/conda_build/cli/main_develop.py @@ -8,7 +8,6 @@ from conda.base.context import context from .. import api -from ..conda_interface import ArgumentParser from .logging import init_logging try: diff --git a/conda_build/cli/main_inspect.py b/conda_build/cli/main_inspect.py index 92c34da2c4..9233020016 100644 --- a/conda_build/cli/main_inspect.py +++ b/conda_build/cli/main_inspect.py @@ -11,7 +11,6 @@ from conda.base.context import context from .. import api -from ..conda_interface import ArgumentParser from .logging import init_logging try: diff --git a/conda_build/cli/main_metapackage.py b/conda_build/cli/main_metapackage.py index da1c56460c..f00319a5d3 100644 --- a/conda_build/cli/main_metapackage.py +++ b/conda_build/cli/main_metapackage.py @@ -9,7 +9,6 @@ from conda.base.context import context from .. import api -from ..conda_interface import ArgumentParser from .logging import init_logging try: diff --git a/conda_build/utils.py b/conda_build/utils.py index 6b77496e67..48c081f7ad 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -69,17 +69,6 @@ from .cli.logging import DuplicateFilter as _DuplicateFilter from .cli.logging import GreaterThanFilter as _GreaterThanFilter from .cli.logging import LessThanFilter as _LessThanFilter -from .conda_interface import ( - PackageRecord, - StringIO, - TemporaryDirectory, - VersionOrder, - cc_conda_build, - download, - unix_path_to_win, - win_path_to_unix, -) -from .conda_interface import rm_rf as _rm_rf from .deprecations import deprecated from .exceptions import BuildLockError From 22930e6b1a83962979ba202971a473b21a56e9ce Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Wed, 4 Sep 2024 16:35:36 -0500 Subject: [PATCH 15/15] Ruff --- conda_build/index.py | 2 +- tests/test_subpackages.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/index.py b/conda_build/index.py index 286552ae98..1b8f4036d0 100644 --- a/conda_build/index.py +++ b/conda_build/index.py @@ -1,7 +1,7 @@ # Copyright (C) 2014 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause -import os import logging +import os from os.path import dirname from conda.base.context import context diff --git a/tests/test_subpackages.py b/tests/test_subpackages.py index e06aeb01b3..84e7503311 100644 --- a/tests/test_subpackages.py +++ b/tests/test_subpackages.py @@ -11,9 +11,9 @@ from conda.base.context import context from conda_build import api, utils +from conda_build.cli.logging import DuplicateFilter from conda_build.exceptions import BuildScriptException, CondaBuildUserError from conda_build.metadata import MetaDataTuple -from conda_build.cli.logging import DuplicateFilter from conda_build.render import finalize_metadata from .utils import get_valid_recipes, subpackage_dir