Merge pull request #225 from lsst/tickets/DM-38065-v25

DM-38065: v25 release notes
lsst · Feb 28, 2023 · dcaf798 · dcaf798
2 parents f60f162 + e2bca95
commit dcaf798
Show file tree

Hide file tree

Showing 19 changed files with 42 additions and 44 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,12 +1,13 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
+      - id: check-toml
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.1.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python
@@ -15,11 +16,11 @@ repos:
         # https://pre-commit.com/#top_level-default_language_version
         language_version: python3.10
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort (python)
   - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
+    rev: 6.0.0
     hooks:
       - id: flake8
diff --git a/doc/changes/DM-35494.feature.md b/doc/changes/DM-35494.feature.md
diff --git a/doc/changes/DM-35917.feature.rst b/doc/changes/DM-35917.feature.rst
diff --git a/doc/changes/DM-36145.feature.md b/doc/changes/DM-36145.feature.md
diff --git a/doc/changes/DM-36412.feature.rst b/doc/changes/DM-36412.feature.rst
diff --git a/doc/changes/DM-36487.api.md b/doc/changes/DM-36487.api.md
diff --git a/doc/changes/DM-36831.feature.md b/doc/changes/DM-36831.feature.md
diff --git a/doc/lsst.ctrl.mpexec/CHANGES.rst b/doc/lsst.ctrl.mpexec/CHANGES.rst
@@ -1,3 +1,31 @@
+lsst-ctrl-mpexec v25.0.0 (2023-02-28)
+=====================================
+
+New Features
+------------
+
+- * Added support for transferring files into execution butler. (`DM-35494 <https://jira.lsstcorp.org/browse/DM-35494>`_)
+- * Added documentation on how to use ``--show`` and ``--config``.
+  * A pipeline will now never execute if ``--show`` option is used with ``pipetask run``.
+  * The ``--config`` option can now accept list configuration values (with or without square brackets), for example ``--config task:listItem=a,b`` or ``--config "task:listItem=[a,b]"``.
+  * The ``--config-file`` option can now take comma-separated file names for multiple config files. (`DM-35917 <https://jira.lsstcorp.org/browse/DM-35917>`_)
+- * added additional quanta information to be displayed by the logger, showing number of quanta per task (`DM-36145 <https://jira.lsstcorp.org/browse/DM-36145>`_)
+- If ``pipetask`` is run with multiple processes and if a butler datastore cache is configured, all subprocesses will now share the same cache.
+  For large numbers of simultaneous processes it may be necessary to significantly increase the number of datasets in the cache to make the cache usable.
+  This can be done by using the ``$DAF_BUTLER_CACHE_EXPIRATION_MODE`` environment variable.
+
+  Previously each subprocess would get its own cache and if ``fork`` start method was used these cache directories would not be cleaned up. (`DM-36412 <https://jira.lsstcorp.org/browse/DM-36412>`_)
+- Always disable implicit threading (e.g. in OpenBLAS) by default in ``pipetask run``, even when not using ``-j``.
+
+  The new ``--enable-implicit-threading`` can be used to turn it back on. (`DM-36831 <https://jira.lsstcorp.org/browse/DM-36831>`_)
+
+
+API Changes
+-----------
+
+- ``SimplePipelineExecutor`` factory methods add ``bind`` parameter for bind values to use with the user expression. (`DM-36487 <https://jira.lsstcorp.org/browse/DM-36487>`_)
+
+
 lsst-ctrl-mpexec v24.0.0 (2022-08-26)
 =====================================
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -20,10 +20,10 @@ classifiers = [
 ]
 keywords = ["lsst"]
 dependencies = [
-    "lsst-utils",
-    "lsst-daf-butler",
-    "lsst-pex-config",
-    "lsst-pipe-base",
+    "lsst-utils >=25.0,<25.100",
+    "lsst-daf-butler >=25.0,<25.100",
+    "lsst-pex-config >=25.0,<25.100",
+    "lsst-pipe-base >=25.0,<25.100",
     "click",
     "astropy",
     "pydantic",

diff --git a/python/lsst/ctrl/mpexec/cli/pipetask.py b/python/lsst/ctrl/mpexec/cli/pipetask.py
@@ -31,7 +31,6 @@
 
 
 class PipetaskCLI(LoaderCLI):
-
     localCmdPkg = "lsst.ctrl.mpexec.cli.cmd"
 
 

diff --git a/python/lsst/ctrl/mpexec/cli/utils.py b/python/lsst/ctrl/mpexec/cli/utils.py
@@ -148,4 +148,4 @@ def makePipelineActions(
 class PipetaskCommand(MWCommand):
     """Command subclass with pipetask-command specific overrides."""
 
-    extra_epilog = "See 'pipetask --help' for more options."  # type: ignore
+    extra_epilog = "See 'pipetask --help' for more options."
diff --git a/python/lsst/ctrl/mpexec/cmdLineFwk.py b/python/lsst/ctrl/mpexec/cmdLineFwk.py
@@ -486,29 +486,23 @@ def makePipeline(self, args: SimpleNamespace) -> Pipeline:
         # loop over all pipeline actions and apply them in order
         for action in args.pipeline_actions:
             if action.action == "add_instrument":
-
                 pipeline.addInstrument(action.value)
 
             elif action.action == "new_task":
-
                 pipeline.addTask(action.value, action.label)
 
             elif action.action == "delete_task":
-
                 pipeline.removeTask(action.label)
 
             elif action.action == "config":
-
                 # action value string is "field=value", split it at '='
                 field, _, value = action.value.partition("=")
                 pipeline.addConfigOverride(action.label, field, value)
 
             elif action.action == "configfile":
-
                 pipeline.addConfigFile(action.label, action.value)
 
             else:
-
                 raise ValueError(f"Unexpected pipeline action: {action.action}")
 
         if args.save_pipeline:

diff --git a/python/lsst/ctrl/mpexec/dotTools.py b/python/lsst/ctrl/mpexec/dotTools.py
@@ -167,10 +167,8 @@ def graph2dot(qgraph: QuantumGraph, file: Any) -> None:
 
     allDatasetRefs: dict[str, str] = {}
     for taskId, taskDef in enumerate(qgraph.taskGraph):
-
         quanta = qgraph.getNodesForTask(taskDef)
         for qId, quantumNode in enumerate(quanta):
-
             # node for a task
             taskNodeName = "task_{}_{}".format(taskId, qId)
             _renderQuantumNode(taskNodeName, taskDef, quantumNode, file)
@@ -254,7 +252,6 @@ def expand_dimensions(connection: connectionTypes.BaseConnection) -> list[str]:
     metadataNodesToLink = set()
 
     for idx, taskDef in enumerate(sorted(pipeline, key=lambda x: x.label)):
-
         # node for a task
         taskNodeName = "task{}".format(idx)
 

diff --git a/python/lsst/ctrl/mpexec/mock_task.py b/python/lsst/ctrl/mpexec/mock_task.py
@@ -119,7 +119,6 @@ def _checkMembership(self, ref: Union[List[DatasetRef], DatasetRef], inout: set)
 
 
 class MockPipelineTaskConfig(PipelineTaskConfig, pipelineConnections=PipelineTaskConnections):
-
     failCondition: Field[str] = Field(
         dtype=str,
         default="",

diff --git a/python/lsst/ctrl/mpexec/mpGraphExecutor.py b/python/lsst/ctrl/mpexec/mpGraphExecutor.py
@@ -456,7 +456,6 @@ def _executeQuantaInProcess(self, graph: QuantumGraph, butler: Butler, report: R
         successCount, totalCount = 0, len(graph)
         failedNodes: set[QuantumNode] = set()
         for qnode in graph:
-
             assert qnode.quantum.dataId is not None, "Quantum DataId cannot be None"
 
             # Any failed inputs mean that the quantum has to be skipped.
@@ -564,7 +563,6 @@ def _executeQuantaMP(self, graph: QuantumGraph, butler: Butler, report: Report)
 
         finishedCount, failedCount = 0, 0
         while jobs.pending or jobs.running:
-
             _LOG.debug("#pendingJobs: %s", len(jobs.pending))
             _LOG.debug("#runningJobs: %s", len(jobs.running))
 

diff --git a/python/lsst/ctrl/mpexec/showInfo.py b/python/lsst/ctrl/mpexec/showInfo.py
@@ -236,7 +236,6 @@ def _showConfigHistory(self, pipeline: Pipeline, showArgs: str) -> None:
 
         found = False
         for taskDef in tasks:
-
             config = taskDef.config
 
             # Look for any matches in the config hierarchy for this name

diff --git a/python/lsst/ctrl/mpexec/singleQuantumExecutor.py b/python/lsst/ctrl/mpexec/singleQuantumExecutor.py
@@ -153,7 +153,6 @@ def _execute(self, taskDef: TaskDef, quantum: Quantum, butler: Butler) -> Quantu
         startTime = time.time()
 
         with self.captureLogging(taskDef, quantum, butler) as captureLog:
-
             # Save detailed resource usage before task start to metadata.
             quantumMetadata = _TASK_METADATA_TYPE()
             logInfo(None, "prep", metadata=quantumMetadata)  # type: ignore

diff --git a/requirements.txt b/requirements.txt
@@ -3,7 +3,7 @@ astropy
 psutil
 pydantic
 networkx
-git+https://github.com/lsst/daf_butler@main#egg=lsst-daf-butler
-git+https://github.com/lsst/utils@main#egg=lsst-utils
-git+https://github.com/lsst/pipe_base@main#egg=lsst-pipe-base
-git+https://github.com/lsst/pex_config@main#egg=lsst-pex-config
+git+https://github.com/lsst/daf_butler@v25.0.x#egg=lsst-daf-butler
+git+https://github.com/lsst/utils@v25.0.x#egg=lsst-utils
+git+https://github.com/lsst/pipe_base@v25.0.x#egg=lsst-pipe-base
+git+https://github.com/lsst/pex_config@v25.0.x#egg=lsst-pex-config
diff --git a/tests/test_cmdLineFwk.py b/tests/test_cmdLineFwk.py
@@ -360,7 +360,6 @@ def testMakeGraphFromSave(self):
         fwk = CmdLineFwk()
 
         with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry() as registryConfig:
-
             # make non-empty graph and store it in a file
             qgraph = _makeQGraph()
             with open(tmpname, "wb") as saveFile:
Original file line number	Diff line number	Diff line change
Expand Up		@@ -31,7 +31,6 @@


		class PipetaskCLI(LoaderCLI):

		localCmdPkg = "lsst.ctrl.mpexec.cli.cmd"


Expand Down