diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml deleted file mode 100644 index 59bae3f1ff..0000000000 --- a/.github/workflows/lint.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: lint - -on: - push: - branches: - - main - pull_request: - -jobs: - call-workflow: - uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main - ruff: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c05804836d..74426fee25 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,25 +7,14 @@ repos: - "--unsafe" - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 - hooks: - - id: black - # It is recommended to specify the latest version of Python - # supported by your project here, or alternatively use - # pre-commit's default_language_version, see - # https://pre-commit.com/#top_level-default_language_version - language_version: python3.11 - - repo: https://github.com/pycqa/isort - rev: 6.0.0 - hooks: - - id: isort - name: isort (python) + - id: check-toml - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.9.3 + rev: v0.9.5 hooks: - id: ruff + args: [--fix] + - id: ruff-format - repo: https://github.com/numpy/numpydoc rev: "v1.8.0" hooks: diff --git a/pyproject.toml b/pyproject.toml index 14bc7d0aa5..7b2fa0bf77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,6 +131,7 @@ target-version = ["py311"] [tool.isort] profile = "black" line_length = 110 +known_first_party = ["lsst"] [tool.lsst_versions] write_to = "python/lsst/daf/butler/version.py" @@ -194,23 +195,33 @@ select = [ "W", # pycodestyle "D", # pydocstyle "UP", # pyupgrade + "I", # isort + "RUF022", # sort __all__ ] # Commented out to suppress "unused noqa" in jenkins which has older ruff not # generating E721. -#extend-select = [ -# "RUF100", # Warn about unused noqa -#] +extend-select = [ + "RUF100", # Warn about unused noqa +] [tool.ruff.lint.per-file-ignores] # parserYacc docstrings can not be fixed. Docstrings are used to define grammar. "python/lsst/daf/butler/registry/queries/expressions/parser/parserYacc.py" = ["D401", "D403"] +[tool.ruff.lint.isort] +known-first-party = ["lsst"] + [tool.ruff.lint.pycodestyle] max-doc-length = 79 [tool.ruff.lint.pydocstyle] convention = "numpy" +[tool.ruff.format] +docstring-code-format = true +# Formatter does not know about indenting. +docstring-code-line-length = 69 + [tool.numpydoc_validation] checks = [ "all", # All except the rules listed below. diff --git a/python/lsst/daf/butler/_butler.py b/python/lsst/daf/butler/_butler.py index b68c148d04..70c6852fba 100644 --- a/python/lsst/daf/butler/_butler.py +++ b/python/lsst/daf/butler/_butler.py @@ -272,8 +272,10 @@ def from_config( "/path/to/repo", run="u/alice/DM-50000/a", collections=[ - "u/alice/DM-50000/a", "u/bob/DM-49998", "HSC/defaults" - ] + "u/alice/DM-50000/a", + "u/bob/DM-49998", + "HSC/defaults", + ], ) This butler will `put` new datasets to the run ``u/alice/DM-50000/a``. @@ -1422,8 +1424,9 @@ def export( with butler.export("exports.yaml") as export: # Export all flats, but none of the dimension element rows # (i.e. data ID information) associated with them. - export.saveDatasets(butler.registry.queryDatasets("flat"), - elements=()) + export.saveDatasets( + butler.registry.queryDatasets("flat"), elements=() + ) # Export all datasets that start with "deepCoadd_" and all of # their associated data ID information. export.saveDatasets(butler.registry.queryDatasets("deepCoadd_*")) diff --git a/python/lsst/daf/butler/_butler_config.py b/python/lsst/daf/butler/_butler_config.py index 4179ee8330..983bcf6f89 100644 --- a/python/lsst/daf/butler/_butler_config.py +++ b/python/lsst/daf/butler/_butler_config.py @@ -26,6 +26,7 @@ # along with this program. If not, see . """Configuration classes specific to the Butler.""" + from __future__ import annotations __all__ = ("ButlerConfig",) diff --git a/python/lsst/daf/butler/_butler_repo_index.py b/python/lsst/daf/butler/_butler_repo_index.py index aca990f7bb..9dbd03ee4b 100644 --- a/python/lsst/daf/butler/_butler_repo_index.py +++ b/python/lsst/daf/butler/_butler_repo_index.py @@ -33,9 +33,10 @@ from typing import Any, ClassVar import yaml -from lsst.resources import ResourcePath from pydantic import TypeAdapter, ValidationError +from lsst.resources import ResourcePath + from ._config import Config from ._utilities.thread_safe_cache import ThreadSafeCache diff --git a/python/lsst/daf/butler/_column_type_info.py b/python/lsst/daf/butler/_column_type_info.py index 55b8e0797e..253cff9e86 100644 --- a/python/lsst/daf/butler/_column_type_info.py +++ b/python/lsst/daf/butler/_column_type_info.py @@ -36,6 +36,7 @@ import astropy.time import sqlalchemy + from lsst.daf.relation import ColumnTag, sql from . import ddl diff --git a/python/lsst/daf/butler/_config.py b/python/lsst/daf/butler/_config.py index a7e19dffb8..aeef8ffd4c 100644 --- a/python/lsst/daf/butler/_config.py +++ b/python/lsst/daf/butler/_config.py @@ -44,9 +44,10 @@ from typing import IO, TYPE_CHECKING, Any, ClassVar, cast import yaml +from yaml.representer import Representer + from lsst.resources import ResourcePath, ResourcePathExpression from lsst.utils import doImportType -from yaml.representer import Representer yaml.add_representer(defaultdict, Representer.represent_dict) diff --git a/python/lsst/daf/butler/_config_support.py b/python/lsst/daf/butler/_config_support.py index f01a860ff3..5d18ffdb05 100644 --- a/python/lsst/daf/butler/_config_support.py +++ b/python/lsst/daf/butler/_config_support.py @@ -29,7 +29,7 @@ from __future__ import annotations -__all__ = ("LookupKey", "processLookupConfigs", "processLookupConfigList") +__all__ = ("LookupKey", "processLookupConfigList", "processLookupConfigs") import logging import re diff --git a/python/lsst/daf/butler/_dataset_ref.py b/python/lsst/daf/butler/_dataset_ref.py index 9e25b98c14..eb1d8a0eb8 100644 --- a/python/lsst/daf/butler/_dataset_ref.py +++ b/python/lsst/daf/butler/_dataset_ref.py @@ -56,9 +56,10 @@ ) import pydantic -from lsst.utils.classes import immutable from pydantic import StrictStr +from lsst.utils.classes import immutable + from ._config_support import LookupKey from ._dataset_type import DatasetType, SerializedDatasetType from ._named import NamedKeyDict diff --git a/python/lsst/daf/butler/_deferredDatasetHandle.py b/python/lsst/daf/butler/_deferredDatasetHandle.py index ab4799c386..cff4e3f718 100644 --- a/python/lsst/daf/butler/_deferredDatasetHandle.py +++ b/python/lsst/daf/butler/_deferredDatasetHandle.py @@ -26,6 +26,7 @@ # along with this program. If not, see . """Module containing classes used with deferring dataset loading.""" + from __future__ import annotations __all__ = ("DeferredDatasetHandle",) diff --git a/python/lsst/daf/butler/_exceptions.py b/python/lsst/daf/butler/_exceptions.py index aff03de7de..70197e5d54 100644 --- a/python/lsst/daf/butler/_exceptions.py +++ b/python/lsst/daf/butler/_exceptions.py @@ -26,6 +26,7 @@ # along with this program. If not, see . """Specialized Butler exceptions.""" + __all__ = ( "ButlerUserError", "CalibrationLookupError", @@ -33,13 +34,13 @@ "CollectionTypeError", "DataIdValueError", "DatasetNotFoundError", - "DimensionNameError", "DatasetTypeNotSupportedError", + "DimensionNameError", "EmptyQueryResultError", "InconsistentDataIdError", "InvalidQueryError", - "MissingDatasetTypeError", "MissingCollectionError", + "MissingDatasetTypeError", "ValidationError", ) @@ -228,9 +229,9 @@ class UnknownButlerUserError(ButlerUserError): UnknownButlerUserError, ) _USER_ERROR_MAPPING = {e.error_type: e for e in _USER_ERROR_TYPES} -assert len(_USER_ERROR_MAPPING) == len( - _USER_ERROR_TYPES -), "Subclasses of ButlerUserError must have unique 'error_type' property" +assert len(_USER_ERROR_MAPPING) == len(_USER_ERROR_TYPES), ( + "Subclasses of ButlerUserError must have unique 'error_type' property" +) def create_butler_user_error(error_type: str, message: str) -> ButlerUserError: diff --git a/python/lsst/daf/butler/_formatter.py b/python/lsst/daf/butler/_formatter.py index bb237e0aac..7b32a7a5e0 100644 --- a/python/lsst/daf/butler/_formatter.py +++ b/python/lsst/daf/butler/_formatter.py @@ -28,13 +28,13 @@ from __future__ import annotations __all__ = ( + "FileIntegrityError", "Formatter", - "FormatterV2", - "FormatterV1inV2", "FormatterFactory", - "FormatterParameter", "FormatterNotImplementedError", - "FileIntegrityError", + "FormatterParameter", + "FormatterV1inV2", + "FormatterV2", ) import contextlib @@ -742,7 +742,6 @@ def read_from_possibly_cached_local_file( self.name(), ), ): - if self.can_read_from_local_file: result = self.read_from_local_file( local_uri.ospath, component=component, expected_size=expected_size diff --git a/python/lsst/daf/butler/_named.py b/python/lsst/daf/butler/_named.py index d21554881b..fdb95b9c11 100644 --- a/python/lsst/daf/butler/_named.py +++ b/python/lsst/daf/butler/_named.py @@ -27,13 +27,13 @@ from __future__ import annotations __all__ = ( + "NameLookupMapping", + "NameMappingSetView", "NamedKeyDict", "NamedKeyMapping", "NamedValueAbstractSet", "NamedValueMutableSet", "NamedValueSet", - "NameLookupMapping", - "NameMappingSetView", ) import contextlib diff --git a/python/lsst/daf/butler/_quantum.py b/python/lsst/daf/butler/_quantum.py index e5efa64e83..38e4a5ecc9 100644 --- a/python/lsst/daf/butler/_quantum.py +++ b/python/lsst/daf/butler/_quantum.py @@ -27,13 +27,14 @@ from __future__ import annotations -__all__ = ("Quantum", "SerializedQuantum", "DimensionRecordsAccumulator") +__all__ = ("DimensionRecordsAccumulator", "Quantum", "SerializedQuantum") import sys from collections.abc import Iterable, Mapping, MutableMapping, Sequence from typing import Any import pydantic + from lsst.utils import doImportType from ._dataset_ref import DatasetRef, SerializedDatasetRef diff --git a/python/lsst/daf/butler/_quantum_backed.py b/python/lsst/daf/butler/_quantum_backed.py index 97ea975e27..59e13ddc81 100644 --- a/python/lsst/daf/butler/_quantum_backed.py +++ b/python/lsst/daf/butler/_quantum_backed.py @@ -39,6 +39,7 @@ from typing import TYPE_CHECKING, Any import pydantic + from lsst.resources import ResourcePath, ResourcePathExpression from ._butler_config import ButlerConfig diff --git a/python/lsst/daf/butler/_storage_class.py b/python/lsst/daf/butler/_storage_class.py index 247ec77b5f..1631c015f4 100644 --- a/python/lsst/daf/butler/_storage_class.py +++ b/python/lsst/daf/butler/_storage_class.py @@ -29,7 +29,7 @@ from __future__ import annotations -__all__ = ("StorageClass", "StorageClassFactory", "StorageClassConfig") +__all__ = ("StorageClass", "StorageClassConfig", "StorageClassFactory") import builtins import itertools diff --git a/python/lsst/daf/butler/_topology.py b/python/lsst/daf/butler/_topology.py index c94e5eb828..96ff97dfd9 100644 --- a/python/lsst/daf/butler/_topology.py +++ b/python/lsst/daf/butler/_topology.py @@ -28,9 +28,9 @@ from __future__ import annotations __all__ = ( - "TopologicalSpace", "TopologicalFamily", "TopologicalRelationshipEndpoint", + "TopologicalSpace", ) import enum diff --git a/python/lsst/daf/butler/arrow_utils.py b/python/lsst/daf/butler/arrow_utils.py index bb23fcccd4..2aad93644d 100644 --- a/python/lsst/daf/butler/arrow_utils.py +++ b/python/lsst/daf/butler/arrow_utils.py @@ -28,15 +28,15 @@ from __future__ import annotations __all__ = ( - "ToArrow", - "RegionArrowType", + "DateTimeArrowScalar", + "DateTimeArrowType", "RegionArrowScalar", - "TimespanArrowType", + "RegionArrowType", "TimespanArrowScalar", - "DateTimeArrowType", - "DateTimeArrowScalar", - "UUIDArrowType", + "TimespanArrowType", + "ToArrow", "UUIDArrowScalar", + "UUIDArrowType", ) import uuid @@ -45,6 +45,7 @@ import astropy.time import pyarrow as pa + from lsst.sphgeom import Region from ._timespan import Timespan diff --git a/python/lsst/daf/butler/cli/butler.py b/python/lsst/daf/butler/cli/butler.py index 1c1a0ad965..fb3c71afdc 100755 --- a/python/lsst/daf/butler/cli/butler.py +++ b/python/lsst/daf/butler/cli/butler.py @@ -27,8 +27,8 @@ from __future__ import annotations __all__ = ( - "LoaderCLI", "ButlerCLI", + "LoaderCLI", "cli", "main", ) @@ -47,6 +47,7 @@ import click import yaml + from lsst.resources import ResourcePath from lsst.utils import doImport from lsst.utils.introspection import get_full_type_name diff --git a/python/lsst/daf/butler/cli/cliLog.py b/python/lsst/daf/butler/cli/cliLog.py index 7fa332007d..cfab0cbe7c 100644 --- a/python/lsst/daf/butler/cli/cliLog.py +++ b/python/lsst/daf/butler/cli/cliLog.py @@ -27,8 +27,8 @@ from __future__ import annotations __all__ = ( - "PrecisionLogFormatter", "CliLog", + "PrecisionLogFormatter", ) import datetime diff --git a/python/lsst/daf/butler/cli/opt/arguments.py b/python/lsst/daf/butler/cli/opt/arguments.py index ffd1f4ecc0..7cc002b3a7 100644 --- a/python/lsst/daf/butler/cli/opt/arguments.py +++ b/python/lsst/daf/butler/cli/opt/arguments.py @@ -29,13 +29,13 @@ __all__ = ( "collection_argument", "collections_argument", + "destination_argument", "dimensions_argument", "directory_argument", "element_argument", "glob_argument", "locations_argument", "repo_argument", - "destination_argument", ) diff --git a/python/lsst/daf/butler/cli/opt/options.py b/python/lsst/daf/butler/cli/opt/options.py index 3056fb7277..8fa8d038c5 100644 --- a/python/lsst/daf/butler/cli/opt/options.py +++ b/python/lsst/daf/butler/cli/opt/options.py @@ -30,36 +30,37 @@ "CollectionTypeCallback", "collection_type_option", "collections_option", - "config_option", "config_file_option", + "config_option", "confirm_option", "dataset_type_option", "datasets_option", - "log_level_option", - "long_log_option", + "limit_option", "log_file_option", "log_label_option", + "log_level_option", "log_tty_option", + "long_log_option", + "offset_option", "options_file_option", + "order_by_option", "processes_option", "regex_option", "register_dataset_types_option", "run_option", + "track_file_attrs_option", "transfer_dimensions_option", "transfer_option", "transfer_option_no_short", "verbose_option", "where_option", - "order_by_option", - "limit_option", - "offset_option", - "track_file_attrs_option", ) from functools import partial from typing import Any import click + from lsst.daf.butler import CollectionType from ..cliLog import CliLog diff --git a/python/lsst/daf/butler/cli/utils.py b/python/lsst/daf/butler/cli/utils.py index 92083a4e97..2b05b9e1e0 100644 --- a/python/lsst/daf/butler/cli/utils.py +++ b/python/lsst/daf/butler/cli/utils.py @@ -27,31 +27,31 @@ from __future__ import annotations __all__ = ( - "astropyTablesToStr", - "printAstropyTables", - "textTypeStr", + "ButlerCommand", "LogCliRunner", + "MWArgument", + "MWArgumentDecorator", + "MWCommand", + "MWCtxObj", + "MWOption", + "MWOptionDecorator", + "MWPath", + "OptionGroup", + "OptionSection", + "addArgumentHelp", + "astropyTablesToStr", + "catch_and_exit", "clickResultMsg", "command_test_env", - "addArgumentHelp", + "option_section", + "printAstropyTables", + "sortAstropyTable", "split_commas", "split_kv", + "textTypeStr", "to_upper", "unwrap", - "option_section", - "MWPath", - "MWOption", - "MWArgument", - "OptionSection", - "MWOptionDecorator", - "MWArgumentDecorator", - "MWCommand", - "ButlerCommand", - "OptionGroup", - "MWCtxObj", "yaml_presets", - "sortAstropyTable", - "catch_and_exit", ) @@ -76,6 +76,7 @@ import click.exceptions import click.testing import yaml + from lsst.utils.iteration import ensure_iterable from .._config import Config @@ -83,6 +84,7 @@ if TYPE_CHECKING: from astropy.table import Table + from lsst.daf.butler import Dimension log = logging.getLogger(__name__) diff --git a/python/lsst/daf/butler/column_spec.py b/python/lsst/daf/butler/column_spec.py index a99874f965..4bf60c190a 100644 --- a/python/lsst/daf/butler/column_spec.py +++ b/python/lsst/daf/butler/column_spec.py @@ -28,17 +28,17 @@ from __future__ import annotations __all__ = ( + "COLLECTION_NAME_MAX_LENGTH", + "BoolColumnSpec", "ColumnSpec", - "IntColumnSpec", - "StringColumnSpec", - "HashColumnSpec", + "ColumnType", "FloatColumnSpec", - "BoolColumnSpec", - "UUIDColumnSpec", + "HashColumnSpec", + "IntColumnSpec", "RegionColumnSpec", + "StringColumnSpec", "TimespanColumnSpec", - "ColumnType", - "COLLECTION_NAME_MAX_LENGTH", + "UUIDColumnSpec", ) import textwrap @@ -49,6 +49,7 @@ import astropy.time import pyarrow as pa import pydantic + from lsst.sphgeom import Region from . import arrow_utils, ddl diff --git a/python/lsst/daf/butler/datastore/_datastore.py b/python/lsst/daf/butler/datastore/_datastore.py index bc30ba6025..1b4296ca51 100644 --- a/python/lsst/daf/butler/datastore/_datastore.py +++ b/python/lsst/daf/butler/datastore/_datastore.py @@ -34,9 +34,9 @@ "Datastore", "DatastoreConfig", "DatastoreOpaqueTable", + "DatastoreTransaction", "DatastoreValidationError", "NullDatastore", - "DatastoreTransaction", ) import contextlib diff --git a/python/lsst/daf/butler/datastore/cache_manager.py b/python/lsst/daf/butler/datastore/cache_manager.py index 13fa16dc8c..9705c3fe6c 100644 --- a/python/lsst/daf/butler/datastore/cache_manager.py +++ b/python/lsst/daf/butler/datastore/cache_manager.py @@ -31,9 +31,9 @@ __all__ = ( "AbstractDatastoreCacheManager", - "DatastoreDisabledCacheManager", "DatastoreCacheManager", "DatastoreCacheManagerConfig", + "DatastoreDisabledCacheManager", ) import atexit @@ -51,9 +51,10 @@ from random import Random from typing import TYPE_CHECKING, Self -from lsst.resources import ResourcePath from pydantic import BaseModel, PrivateAttr +from lsst.resources import ResourcePath + from .._config import Config, ConfigSubset from .._config_support import processLookupConfigs from .._dataset_ref import DatasetId, DatasetRef diff --git a/python/lsst/daf/butler/datastore/constraints.py b/python/lsst/daf/butler/datastore/constraints.py index 28ff4d0e5d..a8ac428133 100644 --- a/python/lsst/daf/butler/datastore/constraints.py +++ b/python/lsst/daf/butler/datastore/constraints.py @@ -31,7 +31,7 @@ from __future__ import annotations -__all__ = ("Constraints", "ConstraintsValidationError", "ConstraintsConfig") +__all__ = ("Constraints", "ConstraintsConfig", "ConstraintsValidationError") import logging from typing import TYPE_CHECKING diff --git a/python/lsst/daf/butler/datastore/file_templates.py b/python/lsst/daf/butler/datastore/file_templates.py index 5496bdf952..2c6b4234a1 100644 --- a/python/lsst/daf/butler/datastore/file_templates.py +++ b/python/lsst/daf/butler/datastore/file_templates.py @@ -29,7 +29,7 @@ from __future__ import annotations -__all__ = ("FileTemplates", "FileTemplate", "FileTemplatesConfig", "FileTemplateValidationError") +__all__ = ("FileTemplate", "FileTemplateValidationError", "FileTemplates", "FileTemplatesConfig") import logging import os.path diff --git a/python/lsst/daf/butler/datastore/stored_file_info.py b/python/lsst/daf/butler/datastore/stored_file_info.py index 513d81e926..50c1f44614 100644 --- a/python/lsst/daf/butler/datastore/stored_file_info.py +++ b/python/lsst/daf/butler/datastore/stored_file_info.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("StoredDatastoreItemInfo", "StoredFileInfo", "SerializedStoredFileInfo") +__all__ = ("SerializedStoredFileInfo", "StoredDatastoreItemInfo", "StoredFileInfo") import inspect from collections.abc import Iterable, Mapping @@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Any import pydantic + from lsst.resources import ResourcePath from lsst.utils import doImportType from lsst.utils.introspection import get_full_type_name diff --git a/python/lsst/daf/butler/datastores/fileDatastore.py b/python/lsst/daf/butler/datastores/fileDatastore.py index 2e945fb80f..26763042ef 100644 --- a/python/lsst/daf/butler/datastores/fileDatastore.py +++ b/python/lsst/daf/butler/datastores/fileDatastore.py @@ -38,6 +38,8 @@ from collections.abc import Callable, Collection, Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, ClassVar, cast +from sqlalchemy import BigInteger, String + from lsst.daf.butler import ( Config, DatasetDatastoreRecords, @@ -105,7 +107,6 @@ # For VERBOSE logging usage. from lsst.utils.logging import VERBOSE, getLogger from lsst.utils.timer import time_this -from sqlalchemy import BigInteger, String if TYPE_CHECKING: from lsst.daf.butler import DatasetProvenance, LookupKey diff --git a/python/lsst/daf/butler/datastores/fileDatastoreClient.py b/python/lsst/daf/butler/datastores/fileDatastoreClient.py index 8f306df4b1..63edb7291e 100644 --- a/python/lsst/daf/butler/datastores/fileDatastoreClient.py +++ b/python/lsst/daf/butler/datastores/fileDatastoreClient.py @@ -1,8 +1,10 @@ -__all__ = ("get_dataset_as_python_object", "FileDatastoreGetPayload") +__all__ = ("FileDatastoreGetPayload", "get_dataset_as_python_object") from typing import Any, Literal import pydantic +from pydantic import AnyHttpUrl + from lsst.daf.butler import DatasetRef, Location from lsst.daf.butler.datastore.cache_manager import ( AbstractDatastoreCacheManager, @@ -15,7 +17,6 @@ generate_datastore_get_information, get_dataset_as_python_object_from_get_info, ) -from pydantic import AnyHttpUrl class FileDatastoreGetPayloadFileInfo(pydantic.BaseModel): diff --git a/python/lsst/daf/butler/datastores/file_datastore/get.py b/python/lsst/daf/butler/datastores/file_datastore/get.py index 795f3256d8..0f93a83030 100644 --- a/python/lsst/daf/butler/datastores/file_datastore/get.py +++ b/python/lsst/daf/butler/datastores/file_datastore/get.py @@ -26,8 +26,8 @@ # along with this program. If not, see . __all__ = ( - "DatastoreFileGetInformation", "DatasetLocationInformation", + "DatastoreFileGetInformation", "generate_datastore_get_information", "get_dataset_as_python_object_from_get_info", ) diff --git a/python/lsst/daf/butler/datastores/file_datastore/retrieve_artifacts.py b/python/lsst/daf/butler/datastores/file_datastore/retrieve_artifacts.py index c143e82712..9802c6b35a 100644 --- a/python/lsst/daf/butler/datastores/file_datastore/retrieve_artifacts.py +++ b/python/lsst/daf/butler/datastores/file_datastore/retrieve_artifacts.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("determine_destination_for_retrieved_artifact", "retrieve_and_zip", "unpack_zips", "ZipIndex") +__all__ = ("ZipIndex", "determine_destination_for_retrieved_artifact", "retrieve_and_zip", "unpack_zips") import logging import tempfile @@ -36,6 +36,8 @@ from collections.abc import Iterable from typing import ClassVar, Literal, Protocol, Self +from pydantic import BaseModel + from lsst.daf.butler import ( DatasetIdFactory, DatasetRef, @@ -44,7 +46,6 @@ ) from lsst.daf.butler.datastore.stored_file_info import SerializedStoredFileInfo from lsst.resources import ResourcePath, ResourcePathExpression -from pydantic import BaseModel _LOG = logging.getLogger(__name__) diff --git a/python/lsst/daf/butler/datastores/inMemoryDatastore.py b/python/lsst/daf/butler/datastores/inMemoryDatastore.py index 132445dbac..bd84a31e9b 100644 --- a/python/lsst/daf/butler/datastores/inMemoryDatastore.py +++ b/python/lsst/daf/butler/datastores/inMemoryDatastore.py @@ -29,7 +29,7 @@ from __future__ import annotations -__all__ = ("StoredMemoryItemInfo", "InMemoryDatastore") +__all__ = ("InMemoryDatastore", "StoredMemoryItemInfo") import logging import time diff --git a/python/lsst/daf/butler/ddl.py b/python/lsst/daf/butler/ddl.py index 007805add9..ffb5979c40 100644 --- a/python/lsst/daf/butler/ddl.py +++ b/python/lsst/daf/butler/ddl.py @@ -36,17 +36,18 @@ instead of "schema" because the latter is too overloaded; in most SQL databases, a "schema" is also another term for a namespace. """ + from __future__ import annotations __all__ = ( - "TableSpec", + "GUID", + "AstropyTimeNsecTai", + "Base64Bytes", + "Base64Region", "FieldSpec", "ForeignKeySpec", "IndexSpec", - "Base64Bytes", - "Base64Region", - "AstropyTimeNsecTai", - "GUID", + "TableSpec", ) import logging @@ -59,9 +60,10 @@ import astropy.time import sqlalchemy +from sqlalchemy.dialects.postgresql import UUID + from lsst.sphgeom import Region from lsst.utils.iteration import ensure_iterable -from sqlalchemy.dialects.postgresql import UUID from . import time_utils from ._config import Config diff --git a/python/lsst/daf/butler/delegates/arrowtable.py b/python/lsst/daf/butler/delegates/arrowtable.py index 1d4c728245..eed21459ea 100644 --- a/python/lsst/daf/butler/delegates/arrowtable.py +++ b/python/lsst/daf/butler/delegates/arrowtable.py @@ -26,6 +26,7 @@ # along with this program. If not, see . """Support for reading Arrow tables.""" + from __future__ import annotations __all__ = ["ArrowTableDelegate"] @@ -34,6 +35,7 @@ from typing import TYPE_CHECKING, Any import pyarrow as pa + from lsst.daf.butler import StorageClassDelegate from lsst.utils.introspection import get_full_type_name from lsst.utils.iteration import ensure_iterable diff --git a/python/lsst/daf/butler/dimensions/_config.py b/python/lsst/daf/butler/dimensions/_config.py index 0209a265d2..5afcf2d067 100644 --- a/python/lsst/daf/butler/dimensions/_config.py +++ b/python/lsst/daf/butler/dimensions/_config.py @@ -34,6 +34,7 @@ from typing import Any, ClassVar, Literal, Union, final import pydantic + from lsst.resources import ResourcePath, ResourcePathExpression from lsst.sphgeom import PixelizationABC from lsst.utils.doImport import doImportType diff --git a/python/lsst/daf/butler/dimensions/_coordinate.py b/python/lsst/daf/butler/dimensions/_coordinate.py index d6f4189e9c..8f5b41d699 100644 --- a/python/lsst/daf/butler/dimensions/_coordinate.py +++ b/python/lsst/daf/butler/dimensions/_coordinate.py @@ -47,6 +47,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, overload import pydantic + from lsst.sphgeom import IntersectionRegion, Region from .._exceptions import DimensionNameError @@ -365,9 +366,9 @@ def from_required_values(dimensions: DimensionGroup, values: tuple[DataIdValue, ``dimensions.implied`` is empty. ``dataId.hasRecords()`` will return `True` if and only if ``dimensions`` is empty. """ - assert len(dimensions.required) == len( - values - ), f"Inconsistency between dimensions {dimensions.required} and required values {values}." + assert len(dimensions.required) == len(values), ( + f"Inconsistency between dimensions {dimensions.required} and required values {values}." + ) if not dimensions: return DataCoordinate.make_empty(dimensions.universe) if not dimensions.implied: @@ -399,9 +400,9 @@ def from_full_values(dimensions: DimensionGroup, values: tuple[DataIdValue, ...] ``dataId.hasRecords()`` will only return `True` if ``dimensions`` is empty. """ - assert len(dimensions) == len( - values - ), f"Inconsistency between dimensions {dimensions.data_coordinate_keys} and full values {values}." + assert len(dimensions) == len(values), ( + f"Inconsistency between dimensions {dimensions.data_coordinate_keys} and full values {values}." + ) if not dimensions: return DataCoordinate.make_empty(dimensions.universe) return _FullTupleDataCoordinate(dimensions, values) diff --git a/python/lsst/daf/butler/dimensions/_data_coordinate_iterable.py b/python/lsst/daf/butler/dimensions/_data_coordinate_iterable.py index 0d5e0e9c23..0f2b78a554 100644 --- a/python/lsst/daf/butler/dimensions/_data_coordinate_iterable.py +++ b/python/lsst/daf/butler/dimensions/_data_coordinate_iterable.py @@ -29,8 +29,8 @@ __all__ = ( "DataCoordinateIterable", - "DataCoordinateSet", "DataCoordinateSequence", + "DataCoordinateSet", ) from abc import abstractmethod diff --git a/python/lsst/daf/butler/dimensions/_elements.py b/python/lsst/daf/butler/dimensions/_elements.py index 8d0f0db91b..f1bef3eaf2 100644 --- a/python/lsst/daf/butler/dimensions/_elements.py +++ b/python/lsst/daf/butler/dimensions/_elements.py @@ -37,9 +37,10 @@ from typing import TYPE_CHECKING, Annotated, Any, ClassVar, TypeAlias, Union, cast import pydantic -from lsst.utils.classes import cached_getter from pydantic_core import core_schema +from lsst.utils.classes import cached_getter + from .. import arrow_utils, column_spec, ddl, pydantic_utils from .._named import NamedValueAbstractSet, NamedValueSet from .._topology import TopologicalRelationshipEndpoint diff --git a/python/lsst/daf/butler/dimensions/_group.py b/python/lsst/daf/butler/dimensions/_group.py index df2342711a..05791d74b1 100644 --- a/python/lsst/daf/butler/dimensions/_group.py +++ b/python/lsst/daf/butler/dimensions/_group.py @@ -36,9 +36,10 @@ import pydantic from deprecated.sphinx import deprecated -from lsst.utils.classes import cached_getter, immutable from pydantic_core import core_schema +from lsst.utils.classes import cached_getter, immutable + from .. import pydantic_utils from .._named import NamedValueAbstractSet, NamedValueSet from .._topology import TopologicalFamily, TopologicalSpace diff --git a/python/lsst/daf/butler/dimensions/_record_set.py b/python/lsst/daf/butler/dimensions/_record_set.py index 051984e3c9..ccdfd33d61 100644 --- a/python/lsst/daf/butler/dimensions/_record_set.py +++ b/python/lsst/daf/butler/dimensions/_record_set.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("DimensionRecordSet", "DimensionRecordFactory") +__all__ = ("DimensionRecordFactory", "DimensionRecordSet") from collections.abc import Collection, Iterable, Iterator from typing import TYPE_CHECKING, Any, Protocol, final diff --git a/python/lsst/daf/butler/dimensions/_record_table.py b/python/lsst/daf/butler/dimensions/_record_table.py index f25436608e..d3b684e1f7 100644 --- a/python/lsst/daf/butler/dimensions/_record_table.py +++ b/python/lsst/daf/butler/dimensions/_record_table.py @@ -34,6 +34,7 @@ import pyarrow as pa import pyarrow.compute as pc + from lsst.utils.iteration import chunk_iterable if TYPE_CHECKING: diff --git a/python/lsst/daf/butler/dimensions/_records.py b/python/lsst/daf/butler/dimensions/_records.py index 278fcb14b5..f04e1acf4d 100644 --- a/python/lsst/daf/butler/dimensions/_records.py +++ b/python/lsst/daf/butler/dimensions/_records.py @@ -32,9 +32,10 @@ from collections.abc import Hashable from typing import TYPE_CHECKING, Any, ClassVar +from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr, create_model + import lsst.sphgeom from lsst.utils.classes import immutable -from pydantic import BaseModel, Field, StrictBool, StrictFloat, StrictInt, StrictStr, create_model from .._timespan import Timespan from ..json import from_json_pydantic, to_json_pydantic @@ -206,9 +207,7 @@ def direct( # readers will read things in as lists. Be kind and transparently # transform to tuples. _recItems = { - k: ( - v if type(v) is not list else Timespan(begin=None, end=None, _nsec=tuple(v)) # noqa: E721 - ) # type: ignore + k: (v if type(v) is not list else Timespan(begin=None, end=None, _nsec=tuple(v))) # type: ignore for k, v in record.items() } diff --git a/python/lsst/daf/butler/dimensions/_schema.py b/python/lsst/daf/butler/dimensions/_schema.py index 8539b057a5..afc8736590 100644 --- a/python/lsst/daf/butler/dimensions/_schema.py +++ b/python/lsst/daf/butler/dimensions/_schema.py @@ -26,7 +26,7 @@ # along with this program. If not, see . from __future__ import annotations -__all__ = ("addDimensionForeignKey", "DimensionRecordSchema") +__all__ = ("DimensionRecordSchema", "addDimensionForeignKey") import copy from collections.abc import Mapping, Set diff --git a/python/lsst/daf/butler/direct_butler/_direct_butler.py b/python/lsst/daf/butler/direct_butler/_direct_butler.py index 973ef25a77..32eef89ccb 100644 --- a/python/lsst/daf/butler/direct_butler/_direct_butler.py +++ b/python/lsst/daf/butler/direct_butler/_direct_butler.py @@ -30,8 +30,8 @@ from __future__ import annotations __all__ = ( - "DirectButler", "ButlerValidationError", + "DirectButler", ) import collections.abc @@ -49,10 +49,11 @@ from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, TextIO, cast from deprecated.sphinx import deprecated +from sqlalchemy.exc import IntegrityError + from lsst.resources import ResourcePath, ResourcePathExpression from lsst.utils.introspection import get_class_of from lsst.utils.logging import VERBOSE, getLogger -from sqlalchemy.exc import IntegrityError from .._butler import Butler from .._butler_config import ButlerConfig diff --git a/python/lsst/daf/butler/direct_butler/_direct_butler_collections.py b/python/lsst/daf/butler/direct_butler/_direct_butler_collections.py index 0d2d549972..2f5d9bb5de 100644 --- a/python/lsst/daf/butler/direct_butler/_direct_butler_collections.py +++ b/python/lsst/daf/butler/direct_butler/_direct_butler_collections.py @@ -33,6 +33,7 @@ from typing import TYPE_CHECKING, Any import sqlalchemy + from lsst.utils.iteration import ensure_iterable from .._butler_collections import ButlerCollections, CollectionInfo diff --git a/python/lsst/daf/butler/direct_query_driver/_driver.py b/python/lsst/daf/butler/direct_query_driver/_driver.py index 486e85e8ea..9834c4c82c 100644 --- a/python/lsst/daf/butler/direct_query_driver/_driver.py +++ b/python/lsst/daf/butler/direct_query_driver/_driver.py @@ -1448,20 +1448,20 @@ def join_dataset_search( # dealt with instead) if materializations start having dataset fields. if union_dataset_type_name is None: dataset_type = self.get_dataset_type(cast(str, resolved_search.name)) - assert ( - dataset_type.name not in joins_builder.fields - ), "Dataset fields have unexpectedly already been joined in." - assert ( - dataset_type.name not in joins_builder.timespans - ), "Dataset timespan has unexpectedly already been joined in." + assert dataset_type.name not in joins_builder.fields, ( + "Dataset fields have unexpectedly already been joined in." + ) + assert dataset_type.name not in joins_builder.timespans, ( + "Dataset timespan has unexpectedly already been joined in." + ) else: dataset_type = self.get_dataset_type(union_dataset_type_name) - assert ( - qt.ANY_DATASET not in joins_builder.fields - ), "Union dataset fields have unexpectedly already been joined in." - assert ( - qt.ANY_DATASET not in joins_builder.timespans - ), "Union dataset timespan has unexpectedly already been joined in." + assert qt.ANY_DATASET not in joins_builder.fields, ( + "Union dataset fields have unexpectedly already been joined in." + ) + assert qt.ANY_DATASET not in joins_builder.timespans, ( + "Union dataset timespan has unexpectedly already been joined in." + ) joins_builder.join( self.managers.datasets.make_joins_builder( @@ -1520,7 +1520,7 @@ def __init__( cursor = self._context.__enter__() try: self._iterator = cursor.partitions() - except: # noqa: E722 + except BaseException: self.close(*sys.exc_info()) raise @@ -1560,6 +1560,6 @@ def next(self) -> ResultPage | None: postprocessed_rows = self._postprocessing.apply(raw_page) return self._page_converter.convert(postprocessed_rows) - except: # noqa: E722 + except BaseException: self.close(*sys.exc_info()) raise diff --git a/python/lsst/daf/butler/direct_query_driver/_postprocessing.py b/python/lsst/daf/butler/direct_query_driver/_postprocessing.py index db577ec6a9..f1abbb3882 100644 --- a/python/lsst/daf/butler/direct_query_driver/_postprocessing.py +++ b/python/lsst/daf/butler/direct_query_driver/_postprocessing.py @@ -33,6 +33,7 @@ from typing import TYPE_CHECKING, ClassVar import sqlalchemy + from lsst.sphgeom import Region from .._exceptions import CalibrationLookupError diff --git a/python/lsst/daf/butler/direct_query_driver/_query_analysis.py b/python/lsst/daf/butler/direct_query_driver/_query_analysis.py index 5b51347fd5..b07b68e2ae 100644 --- a/python/lsst/daf/butler/direct_query_driver/_query_analysis.py +++ b/python/lsst/daf/butler/direct_query_driver/_query_analysis.py @@ -28,10 +28,10 @@ from __future__ import annotations __all__ = ( - "QueryJoinsAnalysis", + "QueryCollectionAnalysis", "QueryFindFirstAnalysis", + "QueryJoinsAnalysis", "ResolvedDatasetSearch", - "QueryCollectionAnalysis", ) import dataclasses diff --git a/python/lsst/daf/butler/direct_query_driver/_result_page_converter.py b/python/lsst/daf/butler/direct_query_driver/_result_page_converter.py index 3c23ad69ef..d3c9456afb 100644 --- a/python/lsst/daf/butler/direct_query_driver/_result_page_converter.py +++ b/python/lsst/daf/butler/direct_query_driver/_result_page_converter.py @@ -282,9 +282,9 @@ def __init__( ctx: ResultPageConverterContext, include_dimension_records: bool, ): - assert ( - list(dimensions.data_coordinate_keys) == ctx.column_order.dimension_key_names - ), "Dimension keys in result row should be in same order as those specified by the result spec" + assert list(dimensions.data_coordinate_keys) == ctx.column_order.dimension_key_names, ( + "Dimension keys in result row should be in same order as those specified by the result spec" + ) self._dimensions = dimensions self._column_order = ctx.column_order diff --git a/python/lsst/daf/butler/direct_query_driver/_sql_builders.py b/python/lsst/daf/butler/direct_query_driver/_sql_builders.py index 3f0d0dba32..0910674ac0 100644 --- a/python/lsst/daf/butler/direct_query_driver/_sql_builders.py +++ b/python/lsst/daf/butler/direct_query_driver/_sql_builders.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("SqlJoinsBuilder", "SqlSelectBuilder", "SqlColumns", "make_table_spec") +__all__ = ("SqlColumns", "SqlJoinsBuilder", "SqlSelectBuilder", "make_table_spec") import dataclasses import itertools @@ -688,7 +688,6 @@ def make_table_spec( def _make_table_indices(dimensions: DimensionGroup) -> list[ddl.IndexSpec]: - index_columns: list[SortedSequenceSet] = [] for dimension in dimensions.required: minimal_group = dimensions.universe[dimension].minimal_group.required diff --git a/python/lsst/daf/butler/formatters/astropyTable.py b/python/lsst/daf/butler/formatters/astropyTable.py index 573a00fe1d..9579bfa027 100644 --- a/python/lsst/daf/butler/formatters/astropyTable.py +++ b/python/lsst/daf/butler/formatters/astropyTable.py @@ -30,6 +30,7 @@ from typing import Any import astropy.table + from lsst.daf.butler import FormatterV2 from lsst.resources import ResourcePath diff --git a/python/lsst/daf/butler/formatters/json.py b/python/lsst/daf/butler/formatters/json.py index 2599831c81..ad27b288ea 100644 --- a/python/lsst/daf/butler/formatters/json.py +++ b/python/lsst/daf/butler/formatters/json.py @@ -34,9 +34,10 @@ import json from typing import Any -from lsst.resources import ResourcePath from pydantic_core import from_json +from lsst.resources import ResourcePath + from .typeless import TypelessFormatter diff --git a/python/lsst/daf/butler/formatters/parquet.py b/python/lsst/daf/butler/formatters/parquet.py index efc12a75fc..2b751345e4 100644 --- a/python/lsst/daf/butler/formatters/parquet.py +++ b/python/lsst/daf/butler/formatters/parquet.py @@ -28,24 +28,24 @@ from __future__ import annotations __all__ = ( + "ArrowAstropySchema", + "ArrowNumpySchema", + "DataFrameSchema", "ParquetFormatter", - "arrow_to_pandas", + "add_pandas_index_to_astropy", + "arrow_schema_to_pandas_index", "arrow_to_astropy", "arrow_to_numpy", "arrow_to_numpy_dict", - "pandas_to_arrow", - "pandas_to_astropy", + "arrow_to_pandas", "astropy_to_arrow", "astropy_to_pandas", - "add_pandas_index_to_astropy", + "compute_row_group_size", + "numpy_dict_to_arrow", "numpy_to_arrow", "numpy_to_astropy", - "numpy_dict_to_arrow", - "arrow_schema_to_pandas_index", - "DataFrameSchema", - "ArrowAstropySchema", - "ArrowNumpySchema", - "compute_row_group_size", + "pandas_to_arrow", + "pandas_to_astropy", ) import collections.abc @@ -58,6 +58,7 @@ import pyarrow as pa import pyarrow.parquet as pq + from lsst.daf.butler import FormatterV2 from lsst.daf.butler.delegates.arrowtable import _checkArrowCompatibleType from lsst.resources import ResourcePath @@ -184,7 +185,6 @@ def _read_parquet( return arrow_table def write_local_file(self, in_memory_dataset: Any, uri: ResourcePath) -> None: - if isinstance(in_memory_dataset, pa.Schema): pq.write_metadata(in_memory_dataset, uri.ospath) return diff --git a/python/lsst/daf/butler/formatters/yaml.py b/python/lsst/daf/butler/formatters/yaml.py index 400c68a156..dfad01b24e 100644 --- a/python/lsst/daf/butler/formatters/yaml.py +++ b/python/lsst/daf/butler/formatters/yaml.py @@ -34,6 +34,7 @@ from typing import Any import yaml + from lsst.resources import ResourcePath from .typeless import TypelessFormatter diff --git a/python/lsst/daf/butler/json.py b/python/lsst/daf/butler/json.py index c59cf27204..d3a0dda758 100644 --- a/python/lsst/daf/butler/json.py +++ b/python/lsst/daf/butler/json.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("to_json_generic", "from_json_generic", "to_json_pydantic", "from_json_pydantic") +__all__ = ("from_json_generic", "from_json_pydantic", "to_json_generic", "to_json_pydantic") import json from typing import TYPE_CHECKING, Any, ClassVar, Protocol diff --git a/python/lsst/daf/butler/logging.py b/python/lsst/daf/butler/logging.py index 0b4bf0f4f9..74476ac849 100644 --- a/python/lsst/daf/butler/logging.py +++ b/python/lsst/daf/butler/logging.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("ButlerMDC", "ButlerLogRecords", "ButlerLogRecordHandler", "ButlerLogRecord", "JsonLogFormatter") +__all__ = ("ButlerLogRecord", "ButlerLogRecordHandler", "ButlerLogRecords", "ButlerMDC", "JsonLogFormatter") import datetime import logging @@ -37,9 +37,10 @@ from logging import Formatter, LogRecord, StreamHandler from typing import IO, Any, ClassVar, overload +from pydantic import BaseModel, ConfigDict, PrivateAttr, RootModel + from lsst.utils.introspection import get_full_type_name from lsst.utils.iteration import isplit -from pydantic import BaseModel, ConfigDict, PrivateAttr, RootModel _LONG_LOG_FORMAT = "{levelname} {asctime} {name} {filename}:{lineno} - {message}" """Default format for log records.""" diff --git a/python/lsst/daf/butler/nonempty_mapping.py b/python/lsst/daf/butler/nonempty_mapping.py index 5e394e8e0a..750504292d 100644 --- a/python/lsst/daf/butler/nonempty_mapping.py +++ b/python/lsst/daf/butler/nonempty_mapping.py @@ -34,7 +34,6 @@ class Copyable(Protocol): - def copy(self) -> Self: ... diff --git a/python/lsst/daf/butler/pydantic_utils.py b/python/lsst/daf/butler/pydantic_utils.py index f8d7595481..5b94b8640e 100644 --- a/python/lsst/daf/butler/pydantic_utils.py +++ b/python/lsst/daf/butler/pydantic_utils.py @@ -29,9 +29,9 @@ __all__ = ( "DeferredValidation", - "get_universe_from_context", "SerializableRegion", "SerializableTime", + "get_universe_from_context", ) from types import EllipsisType @@ -39,9 +39,10 @@ import pydantic from astropy.time import Time -from lsst.sphgeom import Region from pydantic_core import core_schema +from lsst.sphgeom import Region + from .time_utils import TimeConverter if TYPE_CHECKING: @@ -185,9 +186,9 @@ def validated(self, **kwargs: Any) -> _T: def __init_subclass__(cls) -> None: # We override __init_subclass__ to grab the type argument to the # DeferredValidation base class, since that's the wrapped type. - assert ( - cls.__base__ is DeferredValidation - ), "Indirect subclasses of DeferredValidation are not allowed." + assert cls.__base__ is DeferredValidation, ( + "Indirect subclasses of DeferredValidation are not allowed." + ) try: # This uses some typing internals that are not as stable as the # rest of Python, so it's the messiest aspect of this class, but diff --git a/python/lsst/daf/butler/queries/_expression_strings.py b/python/lsst/daf/butler/queries/_expression_strings.py index 50051c7904..e2a351f465 100644 --- a/python/lsst/daf/butler/queries/_expression_strings.py +++ b/python/lsst/daf/butler/queries/_expression_strings.py @@ -31,6 +31,7 @@ from typing import Literal, NamedTuple, TypeAlias import astropy.time + import lsst.sphgeom from .._exceptions import InvalidQueryError @@ -84,9 +85,9 @@ def convert_expression_string_to_predicate( return Predicate.from_bool(True) converter = _ConversionVisitor(context, universe) predicate = tree.visit(converter) - assert isinstance( - predicate, Predicate - ), "The grammar should guarantee that we get a predicate back at the top level." + assert isinstance(predicate, Predicate), ( + "The grammar should guarantee that we get a predicate back at the top level." + ) return predicate @@ -288,7 +289,7 @@ def _make_literal(value: LiteralValue) -> _ColExpr: def _to_timespan_bound(value: _VisitorResult, node: Node) -> astropy.time.Time | None: - match (value): + match value: case _ColExpr(value=expr) if expr.expression_type == "datetime": return expr.value case _Null(): diff --git a/python/lsst/daf/butler/queries/_identifiers.py b/python/lsst/daf/butler/queries/_identifiers.py index 8619e410f9..6e487738eb 100644 --- a/python/lsst/daf/butler/queries/_identifiers.py +++ b/python/lsst/daf/butler/queries/_identifiers.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("interpret_identifier", "IdentifierContext") +__all__ = ("IdentifierContext", "interpret_identifier") import itertools from collections.abc import Mapping, Set diff --git a/python/lsst/daf/butler/queries/_query.py b/python/lsst/daf/butler/queries/_query.py index b6ed527a88..1a6114660a 100644 --- a/python/lsst/daf/butler/queries/_query.py +++ b/python/lsst/daf/butler/queries/_query.py @@ -156,7 +156,7 @@ def expression_factory(self) -> ExpressionFactory: query = query.where( x.instrument == "LSSTCam", x.visit.day_obs > 20240701, - x.any(x.band == 'u', x.band == 'y'), + x.any(x.band == "u", x.band == "y"), ) As shown above, the returned object also has an diff --git a/python/lsst/daf/butler/queries/convert_args.py b/python/lsst/daf/butler/queries/convert_args.py index 16d6a432eb..08d79c41d6 100644 --- a/python/lsst/daf/butler/queries/convert_args.py +++ b/python/lsst/daf/butler/queries/convert_args.py @@ -28,8 +28,8 @@ from __future__ import annotations __all__ = ( - "convert_where_args", "convert_order_by_args", + "convert_where_args", ) from collections.abc import Mapping, Set diff --git a/python/lsst/daf/butler/queries/driver.py b/python/lsst/daf/butler/queries/driver.py index a27381a4a1..659d232e8b 100644 --- a/python/lsst/daf/butler/queries/driver.py +++ b/python/lsst/daf/butler/queries/driver.py @@ -28,12 +28,12 @@ from __future__ import annotations __all__ = ( - "QueryDriver", - "ResultPage", "DataCoordinateResultPage", - "DimensionRecordResultPage", "DatasetRefResultPage", + "DimensionRecordResultPage", "GeneralResultPage", + "QueryDriver", + "ResultPage", ) import dataclasses diff --git a/python/lsst/daf/butler/queries/expression_factory.py b/python/lsst/daf/butler/queries/expression_factory.py index 2ff1822bd1..a3ba3765e9 100644 --- a/python/lsst/daf/butler/queries/expression_factory.py +++ b/python/lsst/daf/butler/queries/expression_factory.py @@ -27,13 +27,14 @@ from __future__ import annotations -__all__ = ("ExpressionFactory", "ExpressionProxy", "ScalarExpressionProxy", "TimespanProxy", "RegionProxy") +__all__ = ("ExpressionFactory", "ExpressionProxy", "RegionProxy", "ScalarExpressionProxy", "TimespanProxy") from abc import ABC, abstractmethod from collections.abc import Iterable from typing import TYPE_CHECKING import astropy.time + from lsst.sphgeom import Region from .._exceptions import InvalidQueryError diff --git a/python/lsst/daf/butler/queries/result_specs.py b/python/lsst/daf/butler/queries/result_specs.py index 462fc12bf5..74f061eb41 100644 --- a/python/lsst/daf/butler/queries/result_specs.py +++ b/python/lsst/daf/butler/queries/result_specs.py @@ -28,10 +28,10 @@ from __future__ import annotations __all__ = ( - "ResultSpecBase", "DataCoordinateResultSpec", - "DimensionRecordResultSpec", "DatasetRefResultSpec", + "DimensionRecordResultSpec", + "ResultSpecBase", ) from abc import ABC, abstractmethod @@ -237,8 +237,7 @@ def find_first_dataset(self) -> str | None: if self.find_first: if len(self.dataset_fields) != 1: raise InvalidQueryError( - "General query with find_first=True cannot have results from multiple " - "dataset searches." + "General query with find_first=True cannot have results from multiple dataset searches." ) (dataset_type,) = self.dataset_fields.keys() return dataset_type diff --git a/python/lsst/daf/butler/queries/tree/_base.py b/python/lsst/daf/butler/queries/tree/_base.py index 953143284e..b327ba1650 100644 --- a/python/lsst/daf/butler/queries/tree/_base.py +++ b/python/lsst/daf/butler/queries/tree/_base.py @@ -28,13 +28,13 @@ from __future__ import annotations __all__ = ( - "QueryTreeBase", + "ANY_DATASET", + "DATASET_FIELD_NAMES", + "AnyDatasetType", "ColumnExpressionBase", "DatasetFieldName", - "DATASET_FIELD_NAMES", + "QueryTreeBase", "is_dataset_field", - "AnyDatasetType", - "ANY_DATASET", ) import enum diff --git a/python/lsst/daf/butler/queries/tree/_column_expression.py b/python/lsst/daf/butler/queries/tree/_column_expression.py index 61b2642efc..01e011c52b 100644 --- a/python/lsst/daf/butler/queries/tree/_column_expression.py +++ b/python/lsst/daf/butler/queries/tree/_column_expression.py @@ -28,13 +28,13 @@ from __future__ import annotations __all__ = ( + "BinaryExpression", + "BinaryOperator", "ColumnExpression", "OrderExpression", - "UnaryExpression", - "BinaryExpression", "Reversed", + "UnaryExpression", "UnaryOperator", - "BinaryOperator", "is_one_datetime_and_one_ingest_date", "is_one_timespan_and_one_datetime", "validate_order_expression", diff --git a/python/lsst/daf/butler/queries/tree/_column_literal.py b/python/lsst/daf/butler/queries/tree/_column_literal.py index 4b6cab6f43..59dff4b79a 100644 --- a/python/lsst/daf/butler/queries/tree/_column_literal.py +++ b/python/lsst/daf/butler/queries/tree/_column_literal.py @@ -44,6 +44,7 @@ import astropy.coordinates import astropy.time import erfa + import lsst.sphgeom from ..._timespan import Timespan diff --git a/python/lsst/daf/butler/queries/tree/_column_reference.py b/python/lsst/daf/butler/queries/tree/_column_reference.py index 91a9fe2b14..60ffc38258 100644 --- a/python/lsst/daf/butler/queries/tree/_column_reference.py +++ b/python/lsst/daf/butler/queries/tree/_column_reference.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("ColumnReference", "DimensionKeyReference", "DimensionFieldReference", "DatasetFieldReference") +__all__ = ("ColumnReference", "DatasetFieldReference", "DimensionFieldReference", "DimensionKeyReference") from typing import TYPE_CHECKING, ClassVar, Literal, TypeAlias, TypeVar, Union, final diff --git a/python/lsst/daf/butler/queries/tree/_column_set.py b/python/lsst/daf/butler/queries/tree/_column_set.py index c30d872afc..ff8013dabe 100644 --- a/python/lsst/daf/butler/queries/tree/_column_set.py +++ b/python/lsst/daf/butler/queries/tree/_column_set.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("ColumnSet", "ColumnOrder", "ResultColumn") +__all__ = ("ColumnOrder", "ColumnSet", "ResultColumn") from collections.abc import Iterable, Iterator, Mapping, Sequence, Set from typing import NamedTuple, cast diff --git a/python/lsst/daf/butler/queries/tree/_predicate.py b/python/lsst/daf/butler/queries/tree/_predicate.py index 476f831ecc..b65fce2ae8 100644 --- a/python/lsst/daf/butler/queries/tree/_predicate.py +++ b/python/lsst/daf/butler/queries/tree/_predicate.py @@ -28,11 +28,11 @@ from __future__ import annotations __all__ = ( + "ComparisonOperator", + "LogicalNotOperand", "Predicate", "PredicateLeaf", - "LogicalNotOperand", "PredicateOperands", - "ComparisonOperator", ) import itertools diff --git a/python/lsst/daf/butler/queries/tree/_query_tree.py b/python/lsst/daf/butler/queries/tree/_query_tree.py index 7d221c73e3..b153f2eb04 100644 --- a/python/lsst/daf/butler/queries/tree/_query_tree.py +++ b/python/lsst/daf/butler/queries/tree/_query_tree.py @@ -28,12 +28,12 @@ from __future__ import annotations __all__ = ( - "QueryTree", - "make_identity_query_tree", "DataCoordinateUploadKey", - "MaterializationKey", "DatasetSearch", + "MaterializationKey", + "QueryTree", "SerializedQueryTree", + "make_identity_query_tree", ) import uuid diff --git a/python/lsst/daf/butler/queries/visitors.py b/python/lsst/daf/butler/queries/visitors.py index 5ee2979b4d..95bc9598fa 100644 --- a/python/lsst/daf/butler/queries/visitors.py +++ b/python/lsst/daf/butler/queries/visitors.py @@ -29,9 +29,9 @@ __all__ = ( "ColumnExpressionVisitor", + "PredicateVisitFlags", "PredicateVisitor", "SimplePredicateVisitor", - "PredicateVisitFlags", ) import enum diff --git a/python/lsst/daf/butler/registry/_config.py b/python/lsst/daf/butler/registry/_config.py index b9c11dc7fd..7b6f58e836 100644 --- a/python/lsst/daf/butler/registry/_config.py +++ b/python/lsst/daf/butler/registry/_config.py @@ -40,6 +40,7 @@ if TYPE_CHECKING: import sqlalchemy + from lsst.resources import ResourcePathExpression diff --git a/python/lsst/daf/butler/registry/_registry.py b/python/lsst/daf/butler/registry/_registry.py index 590eea03bb..faa909a163 100644 --- a/python/lsst/daf/butler/registry/_registry.py +++ b/python/lsst/daf/butler/registry/_registry.py @@ -27,7 +27,7 @@ from __future__ import annotations -__all__ = ("Registry", "CollectionArgType") +__all__ = ("CollectionArgType", "Registry") import contextlib import logging diff --git a/python/lsst/daf/butler/registry/bridge/monolithic.py b/python/lsst/daf/butler/registry/bridge/monolithic.py index 14ce7594d2..0e77729c4a 100644 --- a/python/lsst/daf/butler/registry/bridge/monolithic.py +++ b/python/lsst/daf/butler/registry/bridge/monolithic.py @@ -28,7 +28,7 @@ from ... import ddl -__all__ = ("MonolithicDatastoreRegistryBridgeManager", "MonolithicDatastoreRegistryBridge") +__all__ = ("MonolithicDatastoreRegistryBridge", "MonolithicDatastoreRegistryBridgeManager") import copy from collections import namedtuple @@ -215,9 +215,9 @@ def emptyTrash( if records_table is None: raise ValueError("This implementation requires a records table.") - assert isinstance( - records_table, ByNameOpaqueTableStorage - ), f"Records table must support hidden attributes. Got {type(records_table)}." + assert isinstance(records_table, ByNameOpaqueTableStorage), ( + f"Records table must support hidden attributes. Got {type(records_table)}." + ) if record_class is None: raise ValueError("Record class must be provided if records table is given.") diff --git a/python/lsst/daf/butler/registry/collections/_base.py b/python/lsst/daf/butler/registry/collections/_base.py index 6580dc0dc8..abd05e6d79 100644 --- a/python/lsst/daf/butler/registry/collections/_base.py +++ b/python/lsst/daf/butler/registry/collections/_base.py @@ -36,6 +36,7 @@ from typing import TYPE_CHECKING, Any, Generic, Literal, NamedTuple, TypeVar, cast import sqlalchemy + from lsst.utils.iteration import chunk_iterable from ..._collection_type import CollectionType @@ -688,7 +689,7 @@ def _find_position_in_collection_chain(self, chain_key: K, begin_or_end: Literal table = self._tables.collection_chain func: sqlalchemy.Function - match (begin_or_end): + match begin_or_end: case "begin": func = sqlalchemy.func.min(table.c.position) case "end": diff --git a/python/lsst/daf/butler/registry/connectionString.py b/python/lsst/daf/butler/registry/connectionString.py index 82965f3dc3..b0eba84851 100644 --- a/python/lsst/daf/butler/registry/connectionString.py +++ b/python/lsst/daf/butler/registry/connectionString.py @@ -31,9 +31,10 @@ from typing import TYPE_CHECKING -from lsst.utils.db_auth import DbAuth, DbAuthNotFoundError from sqlalchemy.engine import url +from lsst.utils.db_auth import DbAuth, DbAuthNotFoundError + if TYPE_CHECKING: from ._config import RegistryConfig diff --git a/python/lsst/daf/butler/registry/datasets/byDimensions/_manager.py b/python/lsst/daf/butler/registry/datasets/byDimensions/_manager.py index 9d28b7d45b..277d40430b 100644 --- a/python/lsst/daf/butler/registry/datasets/byDimensions/_manager.py +++ b/python/lsst/daf/butler/registry/datasets/byDimensions/_manager.py @@ -10,6 +10,7 @@ import astropy.time import sqlalchemy + from lsst.daf.relation import Relation, sql from .... import ddl @@ -458,7 +459,7 @@ def getDatasetRef(self, id: DatasetId) -> DatasetRef | None: ) with self._db.query(data_id_sql) as sql_result: data_id_row = sql_result.mappings().fetchone() - assert data_id_row is not None, "Data ID should be present if dataset is." "" + assert data_id_row is not None, "Data ID should be present if dataset is." data_id = DataCoordinate.from_required_values( record.dataset_type.dimensions, tuple(data_id_row[dimension] for dimension in record.dataset_type.dimensions.required), @@ -561,9 +562,9 @@ def _find_storage(self, name: str) -> _DatasetRecordStorage: dataset_type, dataset_type_id = self._cache.get(name) if dataset_type is not None: tables = self._cache.get_by_dimensions(dataset_type.dimensions) - assert ( - dataset_type_id is not None and tables is not None - ), "Dataset type cache population is incomplete." + assert dataset_type_id is not None and tables is not None, ( + "Dataset type cache population is incomplete." + ) return _DatasetRecordStorage( dataset_type=dataset_type, dataset_type_id=dataset_type_id, dynamic_tables=tables ) @@ -576,9 +577,9 @@ def _find_storage(self, name: str) -> _DatasetRecordStorage: dataset_type, dataset_type_id = self._cache.get(name) if dataset_type is not None: tables = self._cache.get_by_dimensions(dataset_type.dimensions) - assert ( - dataset_type_id is not None and tables is not None - ), "Dataset type cache population is incomplete." + assert dataset_type_id is not None and tables is not None, ( + "Dataset type cache population is incomplete." + ) return _DatasetRecordStorage( dataset_type=dataset_type, dataset_type_id=dataset_type_id, dynamic_tables=tables ) @@ -1581,8 +1582,7 @@ def _finish_query_builder( sql_projection.distinct = ( # If there are multiple collections, this subquery might have # non-unique rows. - len(collections) > 1 - and not fields + len(collections) > 1 and not fields ) return sql_projection diff --git a/python/lsst/daf/butler/registry/datasets/byDimensions/summaries.py b/python/lsst/daf/butler/registry/datasets/byDimensions/summaries.py index d57636899e..03dceb51af 100644 --- a/python/lsst/daf/butler/registry/datasets/byDimensions/summaries.py +++ b/python/lsst/daf/butler/registry/datasets/byDimensions/summaries.py @@ -36,6 +36,7 @@ from typing import Any, Generic, TypeVar import sqlalchemy + from lsst.utils.iteration import chunk_iterable from ...._collection_type import CollectionType diff --git a/python/lsst/daf/butler/registry/datasets/byDimensions/tables.py b/python/lsst/daf/butler/registry/datasets/byDimensions/tables.py index 8864cc451f..b3cb7631e7 100644 --- a/python/lsst/daf/butler/registry/datasets/byDimensions/tables.py +++ b/python/lsst/daf/butler/registry/datasets/byDimensions/tables.py @@ -28,19 +28,20 @@ from __future__ import annotations __all__ = ( + "StaticDatasetTablesTuple", "addDatasetForeignKey", "makeCalibTableName", "makeCalibTableSpec", "makeStaticTableSpecs", "makeTagTableName", "makeTagTableSpec", - "StaticDatasetTablesTuple", ) from collections import namedtuple from typing import Any, TypeAlias import sqlalchemy + from lsst.utils.classes import immutable from .... import ddl @@ -634,9 +635,9 @@ def calibs( table : `sqlalchemy.Table` SQLAlchemy table object. """ - assert ( - self.calibs_name is not None - ), "Dataset type should be checked to be calibration by calling code." + assert self.calibs_name is not None, ( + "Dataset type should be checked to be calibration by calling code." + ) table = cache.get(self.calibs_name) if table is not None: return table diff --git a/python/lsst/daf/butler/registry/dimensions/static.py b/python/lsst/daf/butler/registry/dimensions/static.py index bfd6cbcf2e..3e613b6376 100644 --- a/python/lsst/daf/butler/registry/dimensions/static.py +++ b/python/lsst/daf/butler/registry/dimensions/static.py @@ -34,6 +34,7 @@ from typing import TYPE_CHECKING, Any import sqlalchemy + from lsst.daf.relation import Calculation, ColumnExpression, Join, Relation, sql from lsst.sphgeom import Region @@ -1099,9 +1100,9 @@ def visit_spatial_constraint( # out (generally this will require a dataset using that skypix # dimension to be joined in, unless this is the common skypix # system). - assert ( - element.name in self.dimensions - ), "QueryTree guarantees dimensions are expanded when constraints are added." + assert element.name in self.dimensions, ( + "QueryTree guarantees dimensions are expanded when constraints are added." + ) skypix = element case _: raise NotImplementedError( diff --git a/python/lsst/daf/butler/registry/interfaces/_attributes.py b/python/lsst/daf/butler/registry/interfaces/_attributes.py index a2f67c5b13..dc4c2535fb 100644 --- a/python/lsst/daf/butler/registry/interfaces/_attributes.py +++ b/python/lsst/daf/butler/registry/interfaces/_attributes.py @@ -27,8 +27,8 @@ from __future__ import annotations __all__ = [ - "ButlerAttributeManager", "ButlerAttributeExistsError", + "ButlerAttributeManager", ] from abc import abstractmethod diff --git a/python/lsst/daf/butler/registry/interfaces/_bridge.py b/python/lsst/daf/butler/registry/interfaces/_bridge.py index 406074c26d..8e446c456e 100644 --- a/python/lsst/daf/butler/registry/interfaces/_bridge.py +++ b/python/lsst/daf/butler/registry/interfaces/_bridge.py @@ -26,7 +26,7 @@ # along with this program. If not, see . from __future__ import annotations -__all__ = ("DatastoreRegistryBridgeManager", "DatastoreRegistryBridge", "FakeDatasetRef", "DatasetIdRef") +__all__ = ("DatasetIdRef", "DatastoreRegistryBridge", "DatastoreRegistryBridgeManager", "FakeDatasetRef") from abc import ABC, abstractmethod from collections.abc import Iterable diff --git a/python/lsst/daf/butler/registry/interfaces/_database.py b/python/lsst/daf/butler/registry/interfaces/_database.py index 02342c2c6c..8f168cfa89 100644 --- a/python/lsst/daf/butler/registry/interfaces/_database.py +++ b/python/lsst/daf/butler/registry/interfaces/_database.py @@ -30,10 +30,10 @@ __all__ = [ "Database", - "DatabaseMetadata", - "ReadOnlyDatabaseError", "DatabaseConflictError", "DatabaseInsertMode", + "DatabaseMetadata", + "ReadOnlyDatabaseError", "SchemaAlreadyDefinedError", "StaticTablesContext", ] @@ -197,7 +197,8 @@ def addTableTuple(self, specs: tuple[ddl.TableSpec, ...]) -> tuple[sqlalchemy.sc we cannot represent this with type annotations. """ return specs._make( # type: ignore - self.addTable(name, spec) for name, spec in zip(specs._fields, specs, strict=True) # type: ignore + self.addTable(name, spec) + for name, spec in zip(specs._fields, specs, strict=True) # type: ignore ) def addInitializer(self, initializer: Callable[[Database], None]) -> None: @@ -1418,9 +1419,9 @@ def _format_bad(inconsistencies: dict[str, Any]) -> str: f"unique constraint for table {table.name}." ) elif bad: - assert ( - compared is not None - ), "Should not be able to get inconsistencies without comparing." + assert compared is not None, ( + "Should not be able to get inconsistencies without comparing." + ) if inserted: raise RuntimeError( f"Conflict ({bad}) in sync after successful insert; this is " diff --git a/python/lsst/daf/butler/registry/interfaces/_opaque.py b/python/lsst/daf/butler/registry/interfaces/_opaque.py index 6418632c66..cf55882d47 100644 --- a/python/lsst/daf/butler/registry/interfaces/_opaque.py +++ b/python/lsst/daf/butler/registry/interfaces/_opaque.py @@ -31,7 +31,7 @@ from __future__ import annotations -__all__ = ["OpaqueTableStorageManager", "OpaqueTableStorage"] +__all__ = ["OpaqueTableStorage", "OpaqueTableStorageManager"] from abc import ABC, abstractmethod from collections.abc import Iterable, Iterator, Mapping diff --git a/python/lsst/daf/butler/registry/obscore/_config.py b/python/lsst/daf/butler/registry/obscore/_config.py index e7f454d716..cf12e951a0 100644 --- a/python/lsst/daf/butler/registry/obscore/_config.py +++ b/python/lsst/daf/butler/registry/obscore/_config.py @@ -101,8 +101,8 @@ class DatasetTypeConfig(pydantic.BaseModel): """Value for the ``obs_collection`` column, if specified it overrides global value in `ObsCoreConfig`.""" - extra_columns: None | ( - dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] + extra_columns: ( + None | (dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig]) ) = None """Description for additional columns, optional. @@ -152,8 +152,8 @@ class ObsCoreConfig(pydantic.BaseModel): """Mapping of instrument name to facility name. Takes precedence over the ``facility_name``.""" - extra_columns: None | ( - dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig] + extra_columns: ( + None | (dict[str, StrictFloat | StrictInt | StrictBool | StrictStr | ExtraColumnConfig]) ) = None """Description for additional columns, optional. diff --git a/python/lsst/daf/butler/registry/obscore/_manager.py b/python/lsst/daf/butler/registry/obscore/_manager.py index 284f4281d5..d15776b9ff 100644 --- a/python/lsst/daf/butler/registry/obscore/_manager.py +++ b/python/lsst/daf/butler/registry/obscore/_manager.py @@ -37,6 +37,7 @@ from typing import TYPE_CHECKING, Any import sqlalchemy + from lsst.daf.butler import Config, DataCoordinate, DatasetRef, DimensionRecordColumnTag, DimensionUniverse from lsst.daf.relation import Join from lsst.sphgeom import Region @@ -173,9 +174,9 @@ def __init__( self.tagged_collection: str | None = None self.run_patterns: list[re.Pattern] = [] if config.collection_type is ConfigCollectionType.TAGGED: - assert ( - config.collections is not None and len(config.collections) == 1 - ), "Exactly one collection name required for tagged type." + assert config.collections is not None and len(config.collections) == 1, ( + "Exactly one collection name required for tagged type." + ) self.tagged_collection = config.collections[0] elif config.collection_type is ConfigCollectionType.RUN: if config.collections: diff --git a/python/lsst/daf/butler/registry/obscore/_records.py b/python/lsst/daf/butler/registry/obscore/_records.py index 59459e12eb..71dde4c69b 100644 --- a/python/lsst/daf/butler/registry/obscore/_records.py +++ b/python/lsst/daf/butler/registry/obscore/_records.py @@ -38,6 +38,7 @@ from uuid import UUID import astropy.time + from lsst.daf.butler import DataCoordinate, DatasetRef, Dimension, DimensionRecord, DimensionUniverse from lsst.utils.introspection import find_outside_stacklevel, get_full_type_name diff --git a/python/lsst/daf/butler/registry/obscore/_schema.py b/python/lsst/daf/butler/registry/obscore/_schema.py index c0ba5746a4..910c6bc5b3 100644 --- a/python/lsst/daf/butler/registry/obscore/_schema.py +++ b/python/lsst/daf/butler/registry/obscore/_schema.py @@ -34,6 +34,7 @@ from typing import TYPE_CHECKING import sqlalchemy + from lsst.daf.butler import ddl from lsst.utils.iteration import ensure_iterable diff --git a/python/lsst/daf/butler/registry/obscore/default_spatial.py b/python/lsst/daf/butler/registry/obscore/default_spatial.py index b72d2ba117..185b2ba1fb 100644 --- a/python/lsst/daf/butler/registry/obscore/default_spatial.py +++ b/python/lsst/daf/butler/registry/obscore/default_spatial.py @@ -33,6 +33,7 @@ from typing import TYPE_CHECKING, Any import sqlalchemy + from lsst.sphgeom import ConvexPolygon, LonLat, Region from ... import ddl diff --git a/python/lsst/daf/butler/registry/obscore/pgsphere.py b/python/lsst/daf/butler/registry/obscore/pgsphere.py index 809a19b042..4d39ad6818 100644 --- a/python/lsst/daf/butler/registry/obscore/pgsphere.py +++ b/python/lsst/daf/butler/registry/obscore/pgsphere.py @@ -33,10 +33,11 @@ from typing import TYPE_CHECKING, Any import sqlalchemy -from lsst.sphgeom import ConvexPolygon, LonLat, Region from sqlalchemy.dialects.postgresql.base import ischema_names from sqlalchemy.types import UserDefinedType +from lsst.sphgeom import ConvexPolygon, LonLat, Region + from ... import ddl from ._spatial import MissingDatabaseError, RegionTypeError, SpatialObsCorePlugin diff --git a/python/lsst/daf/butler/registry/queries/_results.py b/python/lsst/daf/butler/registry/queries/_results.py index fbb904a0ec..949f5906b3 100644 --- a/python/lsst/daf/butler/registry/queries/_results.py +++ b/python/lsst/daf/butler/registry/queries/_results.py @@ -28,10 +28,10 @@ __all__ = ( "ChainedDatasetQueryResults", + "DataCoordinateQueryResults", "DatabaseDataCoordinateQueryResults", "DatabaseDimensionRecordQueryResults", "DatabaseParentDatasetQueryResults", - "DataCoordinateQueryResults", "DatasetQueryResults", "DimensionRecordQueryResults", "ParentDatasetQueryResults", @@ -47,6 +47,7 @@ from typing import Any, Self from deprecated.sphinx import deprecated + from lsst.utils.introspection import find_outside_stacklevel from ..._dataset_ref import DatasetRef diff --git a/python/lsst/daf/butler/registry/queries/_sql_query_backend.py b/python/lsst/daf/butler/registry/queries/_sql_query_backend.py index 58c10fe9c1..c1a86fc03c 100644 --- a/python/lsst/daf/butler/registry/queries/_sql_query_backend.py +++ b/python/lsst/daf/butler/registry/queries/_sql_query_backend.py @@ -86,9 +86,9 @@ def context(self) -> SqlQueryContext: return SqlQueryContext(self._db, self._managers.column_types) def get_collection_name(self, key: Any) -> str: - assert ( - self._managers.caching_context.collection_records is not None - ), "Collection-record caching should already been enabled any time this is called." + assert self._managers.caching_context.collection_records is not None, ( + "Collection-record caching should already been enabled any time this is called." + ) return self._managers.collections[key].name def resolve_collection_wildcard( diff --git a/python/lsst/daf/butler/registry/queries/_sql_query_context.py b/python/lsst/daf/butler/registry/queries/_sql_query_context.py index 34c5c1a9f3..fd5a726a0c 100644 --- a/python/lsst/daf/butler/registry/queries/_sql_query_context.py +++ b/python/lsst/daf/butler/registry/queries/_sql_query_context.py @@ -34,6 +34,7 @@ from typing import TYPE_CHECKING, Any, cast import sqlalchemy + from lsst.daf.relation import ( BinaryOperationRelation, Calculation, @@ -128,9 +129,9 @@ def count(self, relation: Relation, *, exact: bool = True, discard: bool = False with self._db.query(sql_executable) as sql_result: return cast(int, sql_result.scalar_one()) elif (rows := relation.payload) is not None: - assert isinstance( - rows, iteration.MaterializedRowIterable - ), "Query guarantees that only materialized payloads are attached to its relations." + assert isinstance(rows, iteration.MaterializedRowIterable), ( + "Query guarantees that only materialized payloads are attached to its relations." + ) return len(rows) elif discard: n = 0 @@ -154,9 +155,9 @@ def any(self, relation: Relation, *, execute: bool = True, exact: bool = True) - with self._db.query(sql_executable) as sql_result: return sql_result.one_or_none() is not None elif (rows := relation.payload) is not None: - assert isinstance( - rows, iteration.MaterializedRowIterable - ), "Query guarantees that only materialized payloads are attached to its relations." + assert isinstance(rows, iteration.MaterializedRowIterable), ( + "Query guarantees that only materialized payloads are attached to its relations." + ) return bool(rows) elif execute: for _ in self.fetch_iterable(relation): diff --git a/python/lsst/daf/butler/registry/queries/_structs.py b/python/lsst/daf/butler/registry/queries/_structs.py index 0c062397de..07955b4a26 100644 --- a/python/lsst/daf/butler/registry/queries/_structs.py +++ b/python/lsst/daf/butler/registry/queries/_structs.py @@ -34,6 +34,7 @@ from typing import Any import astropy.time + from lsst.daf.relation import ColumnExpression, ColumnTag, Predicate, SortTerm from lsst.sphgeom import IntersectionRegion, Region from lsst.utils.classes import cached_getter, immutable diff --git a/python/lsst/daf/butler/registry/queries/butler_sql_engine.py b/python/lsst/daf/butler/registry/queries/butler_sql_engine.py index a4d334557d..e0238e02be 100644 --- a/python/lsst/daf/butler/registry/queries/butler_sql_engine.py +++ b/python/lsst/daf/butler/registry/queries/butler_sql_engine.py @@ -36,6 +36,7 @@ import astropy.time import sqlalchemy + from lsst.daf.relation import ColumnTag, Relation, Sort, UnaryOperation, UnaryOperationRelation, sql from ..._column_tags import is_timespan_column diff --git a/python/lsst/daf/butler/registry/queries/expressions/_predicate.py b/python/lsst/daf/butler/registry/queries/expressions/_predicate.py index 7437fa72ec..8084609533 100644 --- a/python/lsst/daf/butler/registry/queries/expressions/_predicate.py +++ b/python/lsst/daf/butler/registry/queries/expressions/_predicate.py @@ -26,7 +26,7 @@ # along with this program. If not, see . from __future__ import annotations -__all__ = ("make_string_expression_predicate", "ExpressionTypeError") +__all__ = ("ExpressionTypeError", "make_string_expression_predicate") import builtins import datetime @@ -37,6 +37,7 @@ import astropy.time import astropy.utils.exceptions + from lsst.daf.relation import ( ColumnContainer, ColumnExpression, @@ -227,9 +228,7 @@ def visitBinaryOp( dtype=b.int | b.float | b.str | astropy.time.Time | datetime.datetime ) as lhs, ColumnExpression() as rhs, - ] if ( - lhs.dtype is rhs.dtype - ): + ] if lhs.dtype is rhs.dtype: return lhs.predicate_method(self.OPERATOR_MAP[operator], rhs) # Allow comparisons between datetime expressions and # astropy.time.Time literals/binds (only), by coercing the @@ -323,9 +322,7 @@ def visitBinaryOp( "+" | "-" | "*", ColumnExpression(dtype=b.int | b.float) as lhs, ColumnExpression() as rhs, - ] if ( - lhs.dtype is rhs.dtype - ): + ] if lhs.dtype is rhs.dtype: return lhs.method(self.OPERATOR_MAP[operator], rhs, dtype=lhs.dtype) case ["/", ColumnExpression(dtype=b.float) as lhs, ColumnExpression(dtype=b.float) as rhs]: return lhs.method("__truediv__", rhs, dtype=b.float) @@ -345,9 +342,9 @@ def visitBinaryOp( ) case ["%", ColumnExpression(dtype=b.int) as lhs, ColumnExpression(dtype=b.int) as rhs]: return lhs.method("__mod__", rhs, dtype=b.int) - assert ( - lhs.dtype is not None and rhs.dtype is not None - ), "Expression converter should not yield untyped nodes." + assert lhs.dtype is not None and rhs.dtype is not None, ( + "Expression converter should not yield untyped nodes." + ) raise ExpressionTypeError( f"Invalid types {lhs.dtype.__name__}, {rhs.dtype.__name__} for binary operator {operator!r} " f"in expression {node!s}." diff --git a/python/lsst/daf/butler/registry/queries/expressions/check.py b/python/lsst/daf/butler/registry/queries/expressions/check.py index 179d71750f..99b93fb6e6 100644 --- a/python/lsst/daf/butler/registry/queries/expressions/check.py +++ b/python/lsst/daf/butler/registry/queries/expressions/check.py @@ -28,8 +28,8 @@ __all__ = ( "CheckVisitor", - "InspectionVisitor", "InspectionSummary", + "InspectionVisitor", ) import dataclasses @@ -45,6 +45,7 @@ if TYPE_CHECKING: import astropy.time + from lsst.daf.relation import ColumnTag diff --git a/python/lsst/daf/butler/registry/queries/expressions/parser/exprTree.py b/python/lsst/daf/butler/registry/queries/expressions/parser/exprTree.py index 839375154a..8b3092e848 100644 --- a/python/lsst/daf/butler/registry/queries/expressions/parser/exprTree.py +++ b/python/lsst/daf/butler/registry/queries/expressions/parser/exprTree.py @@ -38,11 +38,11 @@ from __future__ import annotations __all__ = [ - "Node", "BinaryOp", "FunctionCall", "Identifier", "IsIn", + "Node", "NumericLiteral", "Parens", "PointNode", diff --git a/python/lsst/daf/butler/registry/queries/expressions/parser/parserYacc.py b/python/lsst/daf/butler/registry/queries/expressions/parser/parserYacc.py index bfbfdc599c..3b6c173d63 100644 --- a/python/lsst/daf/butler/registry/queries/expressions/parser/parserYacc.py +++ b/python/lsst/daf/butler/registry/queries/expressions/parser/parserYacc.py @@ -29,7 +29,7 @@ from __future__ import annotations -__all__ = ["ParserYacc", "ParserYaccError", "ParseError", "ParserEOFError"] +__all__ = ["ParseError", "ParserEOFError", "ParserYacc", "ParserYaccError"] import functools import re diff --git a/python/lsst/daf/butler/registry/sql_registry.py b/python/lsst/daf/butler/registry/sql_registry.py index 8e90bbacd9..d4c0d45e24 100644 --- a/python/lsst/daf/butler/registry/sql_registry.py +++ b/python/lsst/daf/butler/registry/sql_registry.py @@ -38,6 +38,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast import sqlalchemy + from lsst.daf.relation import LeafRelation, Relation from lsst.resources import ResourcePathExpression from lsst.utils.iteration import ensure_iterable diff --git a/python/lsst/daf/butler/registry/tests/_database.py b/python/lsst/daf/butler/registry/tests/_database.py index 542d80e640..bca735e635 100644 --- a/python/lsst/daf/butler/registry/tests/_database.py +++ b/python/lsst/daf/butler/registry/tests/_database.py @@ -42,6 +42,7 @@ import astropy.time import sqlalchemy + from lsst.sphgeom import Circle, ConvexPolygon, Mq3cPixelization, UnionRegion, UnitVector3d from ..._timespan import Timespan diff --git a/python/lsst/daf/butler/registry/tests/_registry.py b/python/lsst/daf/butler/registry/tests/_registry.py index 5e5e46bf30..446d152132 100644 --- a/python/lsst/daf/butler/registry/tests/_registry.py +++ b/python/lsst/daf/butler/registry/tests/_registry.py @@ -712,12 +712,12 @@ def testImportDatasetsUUID(self): self.assertEqual(ref2.id, ref1.id) # Cannot import to different run with the same ID - ref = DatasetRef(datasetTypeBias, dataIdBias1, id=ref1.id, run=f"run{run+1}") + ref = DatasetRef(datasetTypeBias, dataIdBias1, id=ref1.id, run=f"run{run + 1}") with self.assertRaises(ConflictingDefinitionError): registry._importDatasets([ref]) ref = DatasetRef( - datasetTypeBias, dataIdBias1, run=f"run{run+1}", id_generation_mode=idGenMode + datasetTypeBias, dataIdBias1, run=f"run{run + 1}", id_generation_mode=idGenMode ) if idGenMode is DatasetIdGenEnum.DATAID_TYPE: # Cannot import same DATAID_TYPE ref into a new run diff --git a/python/lsst/daf/butler/registry/versions.py b/python/lsst/daf/butler/registry/versions.py index a56c10a024..7cb8b30c4e 100644 --- a/python/lsst/daf/butler/registry/versions.py +++ b/python/lsst/daf/butler/registry/versions.py @@ -30,8 +30,8 @@ __all__ = [ "ButlerVersionsManager", "IncompatibleVersionError", - "MissingManagerError", "ManagerMismatchError", + "MissingManagerError", ] import logging diff --git a/python/lsst/daf/butler/remote_butler/_factory.py b/python/lsst/daf/butler/remote_butler/_factory.py index fc7de6ef41..e46fa09b1c 100644 --- a/python/lsst/daf/butler/remote_butler/_factory.py +++ b/python/lsst/daf/butler/remote_butler/_factory.py @@ -30,6 +30,7 @@ __all__ = ("RemoteButlerFactory",) import httpx + from lsst.daf.butler.repo_relocation import replaceRoot from .._butler_config import ButlerConfig diff --git a/python/lsst/daf/butler/remote_butler/_http_connection.py b/python/lsst/daf/butler/remote_butler/_http_connection.py index 83ba11a413..9ee63fda76 100644 --- a/python/lsst/daf/butler/remote_butler/_http_connection.py +++ b/python/lsst/daf/butler/remote_butler/_http_connection.py @@ -38,9 +38,10 @@ from uuid import uuid4 import httpx -from lsst.daf.butler import __version__ from pydantic import BaseModel, ValidationError +from lsst.daf.butler import __version__ + from ._authentication import get_authentication_headers from ._errors import deserialize_butler_user_error from .server_models import CLIENT_REQUEST_ID_HEADER_NAME, ERROR_STATUS_CODE, ErrorResponseModel diff --git a/python/lsst/daf/butler/remote_butler/_remote_butler.py b/python/lsst/daf/butler/remote_butler/_remote_butler.py index 52d9689188..d3956cb1a5 100644 --- a/python/lsst/daf/butler/remote_butler/_remote_butler.py +++ b/python/lsst/daf/butler/remote_butler/_remote_butler.py @@ -37,6 +37,7 @@ from typing import TYPE_CHECKING, Any, TextIO, cast from deprecated.sphinx import deprecated + from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ( ArtifactIndexInfo, ZipIndex, diff --git a/python/lsst/daf/butler/remote_butler/server/_dependencies.py b/python/lsst/daf/butler/remote_butler/server/_dependencies.py index 075d5f5b72..1935ba92ea 100644 --- a/python/lsst/daf/butler/remote_butler/server/_dependencies.py +++ b/python/lsst/daf/butler/remote_butler/server/_dependencies.py @@ -28,6 +28,7 @@ from typing import Annotated from fastapi import Depends + from lsst.daf.butler import LabeledButlerFactory from ._factory import Factory diff --git a/python/lsst/daf/butler/remote_butler/server/handlers/_external_query.py b/python/lsst/daf/butler/remote_butler/server/handlers/_external_query.py index 72f03856cd..7ae178af83 100644 --- a/python/lsst/daf/butler/remote_butler/server/handlers/_external_query.py +++ b/python/lsst/daf/butler/remote_butler/server/handlers/_external_query.py @@ -35,6 +35,7 @@ from fastapi import APIRouter, Depends from fastapi.responses import StreamingResponse + from lsst.daf.butler import Butler, DataCoordinate, DimensionGroup from lsst.daf.butler.remote_butler.server_models import ( DatasetRefResultModel, @@ -198,10 +199,12 @@ def _get_query_context(factory: Factory, query: QueryInputs) -> Iterator[_QueryC allow_duplicate_overlaps=input.allow_duplicate_overlaps, ) elif input.type == "upload": - driver.upload_data_coordinates( - DimensionGroup.from_simple(input.dimensions, butler.dimensions), - [tuple(r) for r in input.rows], - key=input.key, - ), + ( + driver.upload_data_coordinates( + DimensionGroup.from_simple(input.dimensions, butler.dimensions), + [tuple(r) for r in input.rows], + key=input.key, + ), + ) yield _QueryContext(driver=driver, tree=tree) diff --git a/python/lsst/daf/butler/remote_butler/server/handlers/_query_streaming.py b/python/lsst/daf/butler/remote_butler/server/handlers/_query_streaming.py index 6ae5e257cd..8abfc9c6cf 100644 --- a/python/lsst/daf/butler/remote_butler/server/handlers/_query_streaming.py +++ b/python/lsst/daf/butler/remote_butler/server/handlers/_query_streaming.py @@ -35,6 +35,7 @@ from fastapi import HTTPException from fastapi.concurrency import contextmanager_in_threadpool, iterate_in_threadpool from fastapi.responses import StreamingResponse + from lsst.daf.butler.remote_butler.server_models import ( QueryErrorResultModel, QueryExecuteResultData, diff --git a/python/lsst/daf/butler/remote_butler/server_models.py b/python/lsst/daf/butler/remote_butler/server_models.py index b04c935efc..6d9d2ae970 100644 --- a/python/lsst/daf/butler/remote_butler/server_models.py +++ b/python/lsst/daf/butler/remote_butler/server_models.py @@ -32,9 +32,9 @@ "DatasetTypeName", "FindDatasetRequestModel", "FindDatasetResponseModel", - "GetFileResponseModel", "GetCollectionInfoResponseModel", "GetCollectionSummaryResponseModel", + "GetFileResponseModel", ] from collections.abc import Iterable @@ -42,6 +42,7 @@ from uuid import UUID import pydantic + from lsst.daf.butler import ( CollectionInfo, CollectionType, diff --git a/python/lsst/daf/butler/script/ingest_files.py b/python/lsst/daf/butler/script/ingest_files.py index 93dd90b73c..31142e5273 100644 --- a/python/lsst/daf/butler/script/ingest_files.py +++ b/python/lsst/daf/butler/script/ingest_files.py @@ -33,6 +33,7 @@ from typing import TYPE_CHECKING, Any from astropy.table import Table + from lsst.resources import ResourcePath from lsst.utils import doImport diff --git a/python/lsst/daf/butler/script/queryDimensionRecords.py b/python/lsst/daf/butler/script/queryDimensionRecords.py index 272ef5563a..b2575a9a0f 100644 --- a/python/lsst/daf/butler/script/queryDimensionRecords.py +++ b/python/lsst/daf/butler/script/queryDimensionRecords.py @@ -32,6 +32,7 @@ from typing import Any from astropy.table import Table + from lsst.sphgeom import Region from .._butler import Butler @@ -83,7 +84,6 @@ def queryDimensionRecords( butler = Butler.from_config(repo, without_datastore=True) with butler.query() as query: - if datasets: query_collections = collections or "*" dataset_types = butler.registry.queryDatasetTypes(datasets) diff --git a/python/lsst/daf/butler/tests/_datasetsHelper.py b/python/lsst/daf/butler/tests/_datasetsHelper.py index 317f719725..105d9f54a8 100644 --- a/python/lsst/daf/butler/tests/_datasetsHelper.py +++ b/python/lsst/daf/butler/tests/_datasetsHelper.py @@ -28,10 +28,10 @@ from __future__ import annotations __all__ = ( + "BadNoWriteFormatter", + "BadWriteFormatter", "DatasetTestHelper", "DatastoreTestHelper", - "BadWriteFormatter", - "BadNoWriteFormatter", "MultiDetectorFormatter", ) diff --git a/python/lsst/daf/butler/tests/_examplePythonTypes.py b/python/lsst/daf/butler/tests/_examplePythonTypes.py index 8559bdbd58..f58181c302 100644 --- a/python/lsst/daf/butler/tests/_examplePythonTypes.py +++ b/python/lsst/daf/butler/tests/_examplePythonTypes.py @@ -37,9 +37,9 @@ "ListDelegate", "MetricsDelegate", "MetricsExample", - "registerMetricsExample", - "MetricsExampleModel", "MetricsExampleDataclass", + "MetricsExampleModel", + "registerMetricsExample", ) @@ -50,9 +50,10 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any -from lsst.daf.butler import DatasetProvenance, StorageClass, StorageClassDelegate from pydantic import BaseModel +from lsst.daf.butler import DatasetProvenance, StorageClass, StorageClassDelegate + if TYPE_CHECKING: from lsst.daf.butler import Butler, Datastore, FormatterFactory diff --git a/python/lsst/daf/butler/tests/_testRepo.py b/python/lsst/daf/butler/tests/_testRepo.py index e7cdf87f0c..f80e915340 100644 --- a/python/lsst/daf/butler/tests/_testRepo.py +++ b/python/lsst/daf/butler/tests/_testRepo.py @@ -28,12 +28,12 @@ from __future__ import annotations __all__ = [ - "makeTestRepo", - "makeTestCollection", - "addDatasetType", - "expandUniqueId", "DatastoreMock", "addDataIdValue", + "addDatasetType", + "expandUniqueId", + "makeTestCollection", + "makeTestRepo", ] import random @@ -42,6 +42,7 @@ from unittest.mock import MagicMock import sqlalchemy + from lsst.daf.butler import ( Butler, Config, diff --git a/python/lsst/daf/butler/tests/butler_queries.py b/python/lsst/daf/butler/tests/butler_queries.py index 5faaedfae7..f45f8920c9 100644 --- a/python/lsst/daf/butler/tests/butler_queries.py +++ b/python/lsst/daf/butler/tests/butler_queries.py @@ -39,9 +39,10 @@ import astropy.coordinates import astropy.time -from lsst.sphgeom import LonLat, Region from numpy import int64 +from lsst.sphgeom import LonLat, Region + from .._butler import Butler from .._collection_type import CollectionType from .._dataset_ref import DatasetRef @@ -697,7 +698,6 @@ def test_duplicate_overlaps(self) -> None: butler = self.make_butler("base.yaml", "spatial.yaml") butler.registry.defaults = RegistryDefaults(instrument="Cam1", skymap="SkyMap1") with butler.query() as query: - data_ids = list(query.data_ids(["visit", "detector", "patch"]).where(visit=1, detector=1)) self.assertCountEqual( [(data_id["tract"], data_id["patch"]) for data_id in data_ids], [(0, 0), (0, 2)] @@ -940,7 +940,7 @@ def _check_visit_id(query: Query) -> None: list(query.where("patch.region OVERLAPS POINT(0.335, -91)").data_ids(["patch"])) # Negative ra values are allowed. - _check_visit_id(query.where(f"POINT({ra-360}, {dec}) OVERLAPS visit_detector_region.region")) + _check_visit_id(query.where(f"POINT({ra - 360}, {dec}) OVERLAPS visit_detector_region.region")) # Substitute ra and dec values via bind instead of literals in the # string. @@ -1297,15 +1297,15 @@ def test_materialization(self) -> None: # Materialize a spatial-join, which should prevent the creation # of a spatial join in the downstream query. self.check_detector_records( - query.join_dimensions(["visit", "detector", "tract"]).materialize() + query.join_dimensions(["visit", "detector", "tract"]) + .materialize() # The patch constraint here should do nothing, because only the # spatial join from the materialization should exist. The # behavior is surprising no matter what here, and the # recommendation to users is to add an explicit overlap # expression any time it's not obvious what the default is. - .where(skymap="SkyMap1", tract=0, instrument="Cam1", visit=2, patch=5).dimension_records( - "detector" - ), + .where(skymap="SkyMap1", tract=0, instrument="Cam1", visit=2, patch=5) + .dimension_records("detector"), [1, 2], has_postprocessing=True, ) diff --git a/python/lsst/daf/butler/tests/cliLogTestBase.py b/python/lsst/daf/butler/tests/cliLogTestBase.py index d3ad1be05e..4118a35320 100644 --- a/python/lsst/daf/butler/tests/cliLogTestBase.py +++ b/python/lsst/daf/butler/tests/cliLogTestBase.py @@ -49,6 +49,7 @@ from typing import TYPE_CHECKING, Any import click + from lsst.daf.butler.cli.butler import UncachedButlerCLI from lsst.daf.butler.cli.cliLog import CliLog from lsst.daf.butler.cli.opt import ( diff --git a/python/lsst/daf/butler/tests/server.py b/python/lsst/daf/butler/tests/server.py index 7b9d74a8ff..df9b8211b0 100644 --- a/python/lsst/daf/butler/tests/server.py +++ b/python/lsst/daf/butler/tests/server.py @@ -6,6 +6,7 @@ from fastapi import FastAPI, Request from fastapi.testclient import TestClient + from lsst.daf.butler import Butler, Config, LabeledButlerFactory from lsst.daf.butler.remote_butler import RemoteButler, RemoteButlerFactory from lsst.daf.butler.remote_butler.server import create_app @@ -24,7 +25,7 @@ # moto v4 and earlier from moto import mock_s3 as mock_aws # type: ignore -__all__ = ("create_test_server", "TestServerInstance", "TEST_REPOSITORY_NAME") +__all__ = ("TEST_REPOSITORY_NAME", "TestServerInstance", "create_test_server") TEST_REPOSITORY_NAME = "testrepo" diff --git a/python/lsst/daf/butler/tests/testFormatters.py b/python/lsst/daf/butler/tests/testFormatters.py index d71d21a5c1..736b1037a4 100644 --- a/python/lsst/daf/butler/tests/testFormatters.py +++ b/python/lsst/daf/butler/tests/testFormatters.py @@ -28,8 +28,8 @@ from __future__ import annotations __all__ = ( - "FormatterTest", "DoNothingFormatter", + "FormatterTest", "LenientYamlFormatter", "MetricsExampleFormatter", "MultipleExtensionsFormatter", @@ -41,6 +41,7 @@ from typing import TYPE_CHECKING, Any, BinaryIO import yaml + from lsst.resources import ResourceHandleProtocol from .._formatter import Formatter, FormatterV2 diff --git a/python/lsst/daf/butler/transfers/_yaml.py b/python/lsst/daf/butler/transfers/_yaml.py index b460d26072..3b218a3464 100644 --- a/python/lsst/daf/butler/transfers/_yaml.py +++ b/python/lsst/daf/butler/transfers/_yaml.py @@ -39,6 +39,7 @@ import astropy.time import yaml + from lsst.resources import ResourcePath from lsst.utils import doImportType from lsst.utils.introspection import find_outside_stacklevel diff --git a/tests/data/registry/spatial.py b/tests/data/registry/spatial.py index 3cb33b7c01..da0d57bfd7 100644 --- a/tests/data/registry/spatial.py +++ b/tests/data/registry/spatial.py @@ -49,11 +49,13 @@ from collections.abc import Callable, Iterable, Iterator from typing import Any -import lsst.daf.butler # noqa:F401; register Time/YAML conversions. import numpy as np import yaml from astropy.time import Time from astropy.wcs import WCS +from matplotlib import pyplot + +import lsst.daf.butler # noqa:F401; register Time/YAML conversions. from lsst.sphgeom import ( ConvexPolygon, HtmPixelization, @@ -65,7 +67,6 @@ UnitVector3d, Vector3d, ) -from matplotlib import pyplot # Pixelization for which one pixel defines the overall area of interest. PARENT_PIX = Mq3cPixelization(6) diff --git a/tests/test_astropyTableFormatter.py b/tests/test_astropyTableFormatter.py index d0d264dd5f..c93669a651 100644 --- a/tests/test_astropyTableFormatter.py +++ b/tests/test_astropyTableFormatter.py @@ -32,6 +32,7 @@ import numpy from astropy.table import Table + from lsst.daf.butler import Butler, DatasetType from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir diff --git a/tests/test_butler.py b/tests/test_butler.py index bc2ca013ea..a4da62c494 100644 --- a/tests/test_butler.py +++ b/tests/test_butler.py @@ -26,6 +26,7 @@ # along with this program. If not, see . """Tests for Butler.""" + from __future__ import annotations import json @@ -47,6 +48,7 @@ try: import boto3 import botocore + from lsst.resources.s3utils import clean_test_environment_for_s3 try: @@ -67,6 +69,7 @@ def mock_aws(*args: Any, **kwargs: Any) -> Any: # type: ignore[no-untyped-def] create_test_server = None import astropy.time + from lsst.daf.butler import ( Butler, ButlerConfig, diff --git a/tests/test_cliCmdConfigDump.py b/tests/test_cliCmdConfigDump.py index 3aa8a82daa..43307980ea 100644 --- a/tests/test_cliCmdConfigDump.py +++ b/tests/test_cliCmdConfigDump.py @@ -33,6 +33,7 @@ import click import yaml + from lsst.daf.butler.cli import butler from lsst.daf.butler.cli.cmd import config_dump from lsst.daf.butler.cli.opt import options_file_option diff --git a/tests/test_cliCmdIngestFiles.py b/tests/test_cliCmdIngestFiles.py index cb17ec27be..bfd4672b17 100644 --- a/tests/test_cliCmdIngestFiles.py +++ b/tests/test_cliCmdIngestFiles.py @@ -32,6 +32,7 @@ import unittest from astropy.table import Table + from lsst.daf.butler import Butler from lsst.daf.butler.cli.butler import cli from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg diff --git a/tests/test_cliCmdPruneDatasets.py b/tests/test_cliCmdPruneDatasets.py index 3d9b528896..269e3f1724 100644 --- a/tests/test_cliCmdPruneDatasets.py +++ b/tests/test_cliCmdPruneDatasets.py @@ -31,10 +31,11 @@ from itertools import chain from unittest.mock import patch +from astropy.table import Table + # Tests require the SqlRegistry import lsst.daf.butler.registry.sql_registry import lsst.daf.butler.script -from astropy.table import Table from lsst.daf.butler import CollectionInfo, CollectionType from lsst.daf.butler.cli.butler import cli as butlerCli from lsst.daf.butler.cli.cmd.commands import ( diff --git a/tests/test_cliCmdQueryCollections.py b/tests/test_cliCmdQueryCollections.py index e94618a534..a4a510ccd3 100644 --- a/tests/test_cliCmdQueryCollections.py +++ b/tests/test_cliCmdQueryCollections.py @@ -31,6 +31,8 @@ import unittest from astropy.table import Table +from numpy import array + from lsst.daf.butler import Butler, CollectionType from lsst.daf.butler.cli.butler import cli from lsst.daf.butler.cli.cmd import query_collections @@ -38,7 +40,6 @@ from lsst.daf.butler.script import queryCollections from lsst.daf.butler.tests import CliCmdTestBase, DatastoreMock from lsst.daf.butler.tests.utils import ButlerTestHelper, readTable -from numpy import array TESTDIR = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_cliCmdQueryDataIds.py b/tests/test_cliCmdQueryDataIds.py index 323269a042..4bd26c067a 100644 --- a/tests/test_cliCmdQueryDataIds.py +++ b/tests/test_cliCmdQueryDataIds.py @@ -31,10 +31,11 @@ import unittest from astropy.table import Table as AstropyTable +from numpy import array + from lsst.daf.butler import Butler, DatasetType, script from lsst.daf.butler.direct_butler import DirectButler from lsst.daf.butler.tests.utils import ButlerTestHelper, makeTestTempDir, removeTestTempDir -from numpy import array TESTDIR = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_cliCmdQueryDatasetTypes.py b/tests/test_cliCmdQueryDatasetTypes.py index 5436d40f0b..7d9847059e 100644 --- a/tests/test_cliCmdQueryDatasetTypes.py +++ b/tests/test_cliCmdQueryDatasetTypes.py @@ -30,12 +30,13 @@ import unittest from astropy.table import Table as AstropyTable +from numpy import array + from lsst.daf.butler.cli.butler import cli from lsst.daf.butler.cli.cmd import query_dataset_types from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg from lsst.daf.butler.tests import CliCmdTestBase from lsst.daf.butler.tests.utils import ButlerTestHelper, readTable -from numpy import array class QueryDatasetTypesCmdTest(CliCmdTestBase, unittest.TestCase): diff --git a/tests/test_cliCmdQueryDatasets.py b/tests/test_cliCmdQueryDatasets.py index 5a80ea02ae..48b0f7ef70 100644 --- a/tests/test_cliCmdQueryDatasets.py +++ b/tests/test_cliCmdQueryDatasets.py @@ -31,11 +31,12 @@ import unittest from astropy.table import Table as AstropyTable +from numpy import array + from lsst.daf.butler import CollectionType, InvalidQueryError, StorageClassFactory, script from lsst.daf.butler.tests import addDatasetType from lsst.daf.butler.tests.utils import ButlerTestHelper, MetricTestRepo, makeTestTempDir, removeTestTempDir from lsst.resources import ResourcePath -from numpy import array TESTDIR = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_cliCmdQueryDimensionRecords.py b/tests/test_cliCmdQueryDimensionRecords.py index 318ec55fce..e90250e693 100644 --- a/tests/test_cliCmdQueryDimensionRecords.py +++ b/tests/test_cliCmdQueryDimensionRecords.py @@ -31,6 +31,8 @@ import unittest from astropy.table import Table as AstropyTable +from numpy import array + from lsst.daf.butler import Butler, StorageClassFactory from lsst.daf.butler.cli.butler import cli as butlerCli from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg @@ -41,7 +43,6 @@ readTable, removeTestTempDir, ) -from numpy import array TESTDIR = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_cliCmdRemoveCollections.py b/tests/test_cliCmdRemoveCollections.py index 12a8d2d8ad..57558d9075 100644 --- a/tests/test_cliCmdRemoveCollections.py +++ b/tests/test_cliCmdRemoveCollections.py @@ -32,6 +32,8 @@ from collections.abc import Sequence from astropy.table import Table +from numpy import array + from lsst.daf.butler import Butler, CollectionType from lsst.daf.butler.cli.butler import cli as butlerCli from lsst.daf.butler.cli.cmd._remove_collections import ( @@ -51,7 +53,6 @@ readTable, removeTestTempDir, ) -from numpy import array TESTDIR = os.path.abspath(os.path.dirname(__file__)) diff --git a/tests/test_cliCmdRemoveRuns.py b/tests/test_cliCmdRemoveRuns.py index fb86769fd5..0c0b82390e 100644 --- a/tests/test_cliCmdRemoveRuns.py +++ b/tests/test_cliCmdRemoveRuns.py @@ -27,7 +27,6 @@ """Unit tests for daf_butler CLI remove-runs subcommand.""" - import os import unittest diff --git a/tests/test_cliPluginLoader.py b/tests/test_cliPluginLoader.py index 6f630359e5..d9819a42db 100644 --- a/tests/test_cliPluginLoader.py +++ b/tests/test_cliPluginLoader.py @@ -35,6 +35,7 @@ import click import yaml + from lsst.daf.butler.cli import butler, cmd from lsst.daf.butler.cli.butler import UncachedButlerCLI from lsst.daf.butler.cli.utils import LogCliRunner, command_test_env diff --git a/tests/test_cliUtilSplitCommas.py b/tests/test_cliUtilSplitCommas.py index 1d6af58b95..284f630841 100644 --- a/tests/test_cliUtilSplitCommas.py +++ b/tests/test_cliUtilSplitCommas.py @@ -31,6 +31,7 @@ from unittest.mock import MagicMock import click + from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg, split_commas mock = MagicMock() diff --git a/tests/test_cliUtilSplitKv.py b/tests/test_cliUtilSplitKv.py index 6d88b68393..14e5811f6d 100644 --- a/tests/test_cliUtilSplitKv.py +++ b/tests/test_cliUtilSplitKv.py @@ -32,6 +32,7 @@ from unittest.mock import MagicMock import click + from lsst.daf.butler.cli.utils import LogCliRunner, clickResultMsg, split_kv diff --git a/tests/test_cliUtilToUpper.py b/tests/test_cliUtilToUpper.py index b199e22809..60d8f91133 100644 --- a/tests/test_cliUtilToUpper.py +++ b/tests/test_cliUtilToUpper.py @@ -30,6 +30,7 @@ import unittest import click + from lsst.daf.butler.cli.utils import LogCliRunner, to_upper diff --git a/tests/test_cliUtils.py b/tests/test_cliUtils.py index 686aa2d63f..bbc00a331a 100644 --- a/tests/test_cliUtils.py +++ b/tests/test_cliUtils.py @@ -31,6 +31,7 @@ from unittest.mock import MagicMock import click + from lsst.daf.butler.cli.opt import directory_argument, repo_argument from lsst.daf.butler.cli.utils import ( LogCliRunner, @@ -183,7 +184,7 @@ def cmd(things): result = self.runner.invoke(cmd, ["--help"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) expectedOutput = f"""Options: - --things TEXT{' ...' if numberOfArgs != 1 else ''}""" + --things TEXT{" ..." if numberOfArgs != 1 else ""}""" self.assertIn(expectedOutput, result.output) diff --git a/tests/test_config.py b/tests/test_config.py index e7e2a8ebda..0fc7284670 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -43,7 +43,7 @@ def modified_environment(**environ): """Temporarily set environment variables. >>> with modified_environment(DAF_BUTLER_CONFIG_PATHS="/somewhere"): - ... os.environ["DAF_BUTLER_CONFIG_PATHS"] == "/somewhere" + ... os.environ["DAF_BUTLER_CONFIG_PATHS"] == "/somewhere" True >>> "DAF_BUTLER_CONFIG_PATHS" != "/somewhere" diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 794ab8e7e1..80072068ad 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -39,8 +39,9 @@ from collections.abc import Callable, Iterator from typing import Any, cast -import lsst.utils.tests import yaml + +import lsst.utils.tests from lsst.daf.butler import ( Config, DataCoordinate, diff --git a/tests/test_ddl.py b/tests/test_ddl.py index b20af519fd..d0772e19bd 100644 --- a/tests/test_ddl.py +++ b/tests/test_ddl.py @@ -30,6 +30,7 @@ import unittest from astropy.time import Time + from lsst.daf.butler import ddl, time_utils diff --git a/tests/test_dimension_record_containers.py b/tests/test_dimension_record_containers.py index 8906f583c5..e383d4865a 100644 --- a/tests/test_dimension_record_containers.py +++ b/tests/test_dimension_record_containers.py @@ -31,6 +31,7 @@ import pyarrow as pa import pyarrow.parquet as pq + from lsst.daf.butler import DimensionRecordSet, DimensionRecordTable from lsst.daf.butler.tests.utils import create_populated_sqlite_registry diff --git a/tests/test_dimensions.py b/tests/test_dimensions.py index a613572e33..c87bed0687 100644 --- a/tests/test_dimensions.py +++ b/tests/test_dimensions.py @@ -35,8 +35,9 @@ from dataclasses import dataclass from random import Random -import lsst.sphgeom import pydantic + +import lsst.sphgeom from lsst.daf.butler import ( Config, DataCoordinate, diff --git a/tests/test_exprParserYacc.py b/tests/test_exprParserYacc.py index 269e1ac2c0..30cf5a3d06 100644 --- a/tests/test_exprParserYacc.py +++ b/tests/test_exprParserYacc.py @@ -30,6 +30,7 @@ import unittest import astropy.time + from lsst.daf.butler.registry.queries.expressions.parser import ParseError, ParserYacc, TreeVisitor, exprTree from lsst.daf.butler.registry.queries.expressions.parser.parserYacc import _parseTimeString diff --git a/tests/test_expressions.py b/tests/test_expressions.py index a4ad6568bc..b3202b2a3d 100644 --- a/tests/test_expressions.py +++ b/tests/test_expressions.py @@ -30,6 +30,8 @@ import astropy.time import sqlalchemy +from sqlalchemy.schema import Column + from lsst.daf.butler import ( ColumnTypeInfo, DataCoordinate, @@ -44,7 +46,6 @@ from lsst.daf.butler.registry.queries.expressions.parser import ParserYacc from lsst.daf.butler.timespan_database_representation import TimespanDatabaseRepresentation from lsst.daf.relation import ColumnContainer, ColumnExpression -from sqlalchemy.schema import Column class FakeDatasetRecordStorageManager: diff --git a/tests/test_normalFormExpression.py b/tests/test_normalFormExpression.py index 743a815ea7..c23e2aa7ca 100644 --- a/tests/test_normalFormExpression.py +++ b/tests/test_normalFormExpression.py @@ -30,6 +30,7 @@ import unittest import astropy.time + from lsst.daf.butler.registry.queries.expressions.normalForm import ( NormalForm, NormalFormExpression, diff --git a/tests/test_obscore.py b/tests/test_obscore.py index 830282cc00..6c2ce39877 100644 --- a/tests/test_obscore.py +++ b/tests/test_obscore.py @@ -33,6 +33,7 @@ import astropy.time import sqlalchemy + from lsst.daf.butler import ( CollectionType, Config, diff --git a/tests/test_parquet.py b/tests/test_parquet.py index 700789a8a8..87024d78e3 100644 --- a/tests/test_parquet.py +++ b/tests/test_parquet.py @@ -57,6 +57,7 @@ try: import boto3 import botocore + from lsst.resources.s3utils import clean_test_environment_for_s3 try: diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py index cb6d0f2314..4bf356f55d 100644 --- a/tests/test_postgresql.py +++ b/tests/test_postgresql.py @@ -35,6 +35,7 @@ import astropy.time import sqlalchemy + from lsst.daf.butler import Butler, ButlerConfig, StorageClassFactory, Timespan, ddl from lsst.daf.butler.datastore import NullDatastore from lsst.daf.butler.direct_butler import DirectButler diff --git a/tests/test_progress.py b/tests/test_progress.py index 57a7302d4a..fc68059689 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -31,6 +31,7 @@ from contextlib import contextmanager import click + from lsst.daf.butler.cli.progress import ClickProgressHandler from lsst.daf.butler.cli.utils import clickResultMsg from lsst.daf.butler.progress import Progress, ProgressHandler diff --git a/tests/test_pydantic_utils.py b/tests/test_pydantic_utils.py index 43f18c7b07..59e835b004 100644 --- a/tests/test_pydantic_utils.py +++ b/tests/test_pydantic_utils.py @@ -31,6 +31,7 @@ import pydantic from astropy.time import Time + from lsst.daf.butler.pydantic_utils import DeferredValidation, SerializableRegion, SerializableTime from lsst.sphgeom import ConvexPolygon, Mq3cPixelization diff --git a/tests/test_query_interface.py b/tests/test_query_interface.py index a9cb4eba30..81d963b6c4 100644 --- a/tests/test_query_interface.py +++ b/tests/test_query_interface.py @@ -47,6 +47,7 @@ from typing import Any import astropy.time + from lsst.daf.butler import ( CollectionType, DataCoordinate, diff --git a/tests/test_query_relations.py b/tests/test_query_relations.py index af1299ffdc..a7d906a897 100644 --- a/tests/test_query_relations.py +++ b/tests/test_query_relations.py @@ -69,9 +69,9 @@ def setUpClass(cls) -> None: # will be no overall results. data_file = os.path.normpath(os.path.join(TESTDIR, "data", "registry", "hsc-rc2-subset.yaml")) cls.butler = create_populated_sqlite_registry(data_file) - assert ( - cls.butler.dimensions.commonSkyPix.name == "htm7" - ), "If this changes, update the skypix levels below to have one below and one above." + assert cls.butler.dimensions.commonSkyPix.name == "htm7", ( + "If this changes, update the skypix levels below to have one below and one above." + ) cls.htm7 = 222340 cls.htm11 = 56919188 cls.instrument = "HSC" diff --git a/tests/test_query_utilities.py b/tests/test_query_utilities.py index 14060b90f6..143b53bef4 100644 --- a/tests/test_query_utilities.py +++ b/tests/test_query_utilities.py @@ -35,6 +35,7 @@ from collections.abc import Iterable, Set import astropy.time + from lsst.daf.butler import DimensionUniverse, InvalidQueryError, Timespan from lsst.daf.butler.dimensions import DimensionElement, DimensionGroup from lsst.daf.butler.queries import tree as qt diff --git a/tests/test_remote_butler.py b/tests/test_remote_butler.py index 891d40b868..717df94dd0 100644 --- a/tests/test_remote_butler.py +++ b/tests/test_remote_butler.py @@ -29,6 +29,8 @@ import unittest from unittest.mock import patch +from pydantic import ValidationError + from lsst.daf.butler import Butler from lsst.daf.butler._exceptions import UnknownButlerUserError from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import ( @@ -37,10 +39,10 @@ from lsst.daf.butler.registry.tests import RegistryTests from lsst.daf.butler.tests.postgresql import TemporaryPostgresInstance, setup_postgres_test_db from lsst.resources import ResourcePath -from pydantic import ValidationError try: import httpx + from lsst.daf.butler.remote_butler import ButlerServerError, RemoteButler remote_butler_import_fail_message = "" diff --git a/tests/test_server.py b/tests/test_server.py index dc09672964..f80bb8b35e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -38,10 +38,11 @@ try: # Failing to import any of these should disable the tests. - import lsst.daf.butler.remote_butler._query_results - import lsst.daf.butler.remote_butler.server.handlers._query_streaming import safir.dependencies.logger from fastapi.testclient import TestClient + + import lsst.daf.butler.remote_butler._query_results + import lsst.daf.butler.remote_butler.server.handlers._query_streaming from lsst.daf.butler.remote_butler import RemoteButler from lsst.daf.butler.remote_butler._authentication import _EXPLICIT_BUTLER_ACCESS_TOKEN_ENVIRONMENT_KEY from lsst.daf.butler.remote_butler.server import create_app diff --git a/tests/test_simpleButler.py b/tests/test_simpleButler.py index d3364d7af1..56a6db4d50 100644 --- a/tests/test_simpleButler.py +++ b/tests/test_simpleButler.py @@ -41,6 +41,7 @@ np = None import astropy.time + from lsst.daf.butler import ( Butler, ButlerConfig, diff --git a/tests/test_sqlite.py b/tests/test_sqlite.py index db823256a9..e391883a94 100644 --- a/tests/test_sqlite.py +++ b/tests/test_sqlite.py @@ -33,6 +33,7 @@ from contextlib import contextmanager import sqlalchemy + from lsst.daf.butler import Butler, Config, ddl from lsst.daf.butler.registry import _RegistryFactory from lsst.daf.butler.registry.databases.sqlite import SqliteDatabase diff --git a/tests/test_time_utils.py b/tests/test_time_utils.py index 174ec16bb5..66e85e626d 100644 --- a/tests/test_time_utils.py +++ b/tests/test_time_utils.py @@ -39,6 +39,7 @@ import astropy.utils.exceptions from astropy.time import Time, TimeDelta + from lsst.daf.butler.time_utils import TimeConverter diff --git a/tests/test_timespan.py b/tests/test_timespan.py index 5102ceb363..eca706c4c2 100644 --- a/tests/test_timespan.py +++ b/tests/test_timespan.py @@ -40,6 +40,7 @@ erfa = None import pydantic + from lsst.daf.butler import Timespan from lsst.daf.butler.time_utils import TimeConverter diff --git a/tests_integration/test_docker_container.py b/tests_integration/test_docker_container.py index c12e3d31c0..32ac5dc96b 100644 --- a/tests_integration/test_docker_container.py +++ b/tests_integration/test_docker_container.py @@ -5,9 +5,10 @@ import unittest import httpx +from testcontainers.core.container import DockerContainer + from lsst.daf.butler.remote_butler import RemoteButlerFactory from lsst.daf.butler.tests.utils import MetricTestRepo -from testcontainers.core.container import DockerContainer TESTDIR = os.path.abspath(os.path.dirname(__file__))