From d59aabbb58d8c44b79d7095cb74e46c6b8eeff20 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Tue, 28 Feb 2023 15:30:05 -0700 Subject: [PATCH 1/6] v25 release notes --- doc/changes/DM-35494.feature.md | 1 - doc/changes/DM-35917.feature.rst | 4 ---- doc/changes/DM-36145.feature.md | 1 - doc/changes/DM-36412.feature.rst | 5 ----- doc/changes/DM-36487.api.md | 1 - doc/changes/DM-36831.feature.md | 3 --- doc/lsst.ctrl.mpexec/CHANGES.rst | 28 ++++++++++++++++++++++++++++ 7 files changed, 28 insertions(+), 15 deletions(-) delete mode 100644 doc/changes/DM-35494.feature.md delete mode 100644 doc/changes/DM-35917.feature.rst delete mode 100644 doc/changes/DM-36145.feature.md delete mode 100644 doc/changes/DM-36412.feature.rst delete mode 100644 doc/changes/DM-36487.api.md delete mode 100644 doc/changes/DM-36831.feature.md diff --git a/doc/changes/DM-35494.feature.md b/doc/changes/DM-35494.feature.md deleted file mode 100644 index 754fc204..00000000 --- a/doc/changes/DM-35494.feature.md +++ /dev/null @@ -1 +0,0 @@ -* Added support for transferring files into execution butler. diff --git a/doc/changes/DM-35917.feature.rst b/doc/changes/DM-35917.feature.rst deleted file mode 100644 index 231f93bd..00000000 --- a/doc/changes/DM-35917.feature.rst +++ /dev/null @@ -1,4 +0,0 @@ -* Added documentation on how to use ``--show`` and ``--config``. -* A pipeline will now never execute if ``--show`` option is used with ``pipetask run``. -* The ``--config`` option can now accept list configuration values (with or without square brackets), for example ``--config task:listItem=a,b`` or ``--config "task:listItem=[a,b]"``. -* The ``--config-file`` option can now take comma-separated file names for multiple config files. diff --git a/doc/changes/DM-36145.feature.md b/doc/changes/DM-36145.feature.md deleted file mode 100644 index 5ad513ca..00000000 --- a/doc/changes/DM-36145.feature.md +++ /dev/null @@ -1 +0,0 @@ -* added additional quanta information to be displayed by the logger, showing number of quanta per task \ No newline at end of file diff --git a/doc/changes/DM-36412.feature.rst b/doc/changes/DM-36412.feature.rst deleted file mode 100644 index 839cb65c..00000000 --- a/doc/changes/DM-36412.feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -If ``pipetask`` is run with multiple processes and if a butler datastore cache is configured, all subprocesses will now share the same cache. -For large numbers of simultaneous processes it may be necessary to significantly increase the number of datasets in the cache to make the cache usable. -This can be done by using the ``$DAF_BUTLER_CACHE_EXPIRATION_MODE`` environment variable. - -Previously each subprocess would get its own cache and if ``fork`` start method was used these cache directories would not be cleaned up. diff --git a/doc/changes/DM-36487.api.md b/doc/changes/DM-36487.api.md deleted file mode 100644 index 67761327..00000000 --- a/doc/changes/DM-36487.api.md +++ /dev/null @@ -1 +0,0 @@ -`SimplePipelineExecutor` factory methods add `bind` parameter for bind values to use with the user expression. diff --git a/doc/changes/DM-36831.feature.md b/doc/changes/DM-36831.feature.md deleted file mode 100644 index b17fa992..00000000 --- a/doc/changes/DM-36831.feature.md +++ /dev/null @@ -1,3 +0,0 @@ -Always disable implicit threading (e.g. in OpenBLAS) by default in `pipetask run`, even when not using `-j`. - -The new `--enable-implicit-threading` can be used to turn it back on. diff --git a/doc/lsst.ctrl.mpexec/CHANGES.rst b/doc/lsst.ctrl.mpexec/CHANGES.rst index 28836026..f851f436 100644 --- a/doc/lsst.ctrl.mpexec/CHANGES.rst +++ b/doc/lsst.ctrl.mpexec/CHANGES.rst @@ -1,3 +1,31 @@ +lsst-ctrl-mpexec v25.0.0 (2023-02-28) +===================================== + +New Features +------------ + +- * Added support for transferring files into execution butler. (`DM-35494 `_) +- * Added documentation on how to use ``--show`` and ``--config``. + * A pipeline will now never execute if ``--show`` option is used with ``pipetask run``. + * The ``--config`` option can now accept list configuration values (with or without square brackets), for example ``--config task:listItem=a,b`` or ``--config "task:listItem=[a,b]"``. + * The ``--config-file`` option can now take comma-separated file names for multiple config files. (`DM-35917 `_) +- * added additional quanta information to be displayed by the logger, showing number of quanta per task (`DM-36145 `_) +- If ``pipetask`` is run with multiple processes and if a butler datastore cache is configured, all subprocesses will now share the same cache. + For large numbers of simultaneous processes it may be necessary to significantly increase the number of datasets in the cache to make the cache usable. + This can be done by using the ``$DAF_BUTLER_CACHE_EXPIRATION_MODE`` environment variable. + + Previously each subprocess would get its own cache and if ``fork`` start method was used these cache directories would not be cleaned up. (`DM-36412 `_) +- Always disable implicit threading (e.g. in OpenBLAS) by default in ``pipetask run``, even when not using ``-j``. + + The new ``--enable-implicit-threading`` can be used to turn it back on. (`DM-36831 `_) + + +API Changes +----------- + +- ``SimplePipelineExecutor`` factory methods add ``bind`` parameter for bind values to use with the user expression. (`DM-36487 `_) + + lsst-ctrl-mpexec v24.0.0 (2022-08-26) ===================================== From e7be2f35f18e4d547a74418c72d70e361e0b0f28 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Wed, 1 Feb 2023 15:27:12 -0700 Subject: [PATCH 2/6] Update pre-commit dependencies --- .pre-commit-config.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 130342f6..1817ebef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,13 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace + - id: check-toml - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.1.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -15,11 +16,11 @@ repos: # https://pre-commit.com/#top_level-default_language_version language_version: python3.10 - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort name: isort (python) - repo: https://github.com/PyCQA/flake8 - rev: 4.0.1 + rev: 6.0.0 hooks: - id: flake8 From bac09df80b1043eda98a45a4b1efc43205cb59b8 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Tue, 28 Feb 2023 15:31:31 -0700 Subject: [PATCH 3/6] Pin dependencies to v25 release branches --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 901de991..1a7ad7bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ astropy psutil pydantic networkx -git+https://github.com/lsst/daf_butler@main#egg=lsst-daf-butler -git+https://github.com/lsst/utils@main#egg=lsst-utils -git+https://github.com/lsst/pipe_base@main#egg=lsst-pipe-base -git+https://github.com/lsst/pex_config@main#egg=lsst-pex-config +git+https://github.com/lsst/daf_butler@v25.0.x#egg=lsst-daf-butler +git+https://github.com/lsst/utils@v25.0.x#egg=lsst-utils +git+https://github.com/lsst/pipe_base@v25.0.x#egg=lsst-pipe-base +git+https://github.com/lsst/pex_config@v25.0.x#egg=lsst-pex-config From 53601b88b71f54a2db4107ee3581d5cea71013f3 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Tue, 28 Feb 2023 15:33:05 -0700 Subject: [PATCH 4/6] Updates for new black --- python/lsst/ctrl/mpexec/cli/pipetask.py | 1 - python/lsst/ctrl/mpexec/cmdLineFwk.py | 6 ------ python/lsst/ctrl/mpexec/dotTools.py | 3 --- python/lsst/ctrl/mpexec/mock_task.py | 1 - python/lsst/ctrl/mpexec/mpGraphExecutor.py | 2 -- python/lsst/ctrl/mpexec/showInfo.py | 1 - python/lsst/ctrl/mpexec/singleQuantumExecutor.py | 1 - tests/test_cmdLineFwk.py | 1 - 8 files changed, 16 deletions(-) diff --git a/python/lsst/ctrl/mpexec/cli/pipetask.py b/python/lsst/ctrl/mpexec/cli/pipetask.py index 5d636b41..142c445e 100644 --- a/python/lsst/ctrl/mpexec/cli/pipetask.py +++ b/python/lsst/ctrl/mpexec/cli/pipetask.py @@ -31,7 +31,6 @@ class PipetaskCLI(LoaderCLI): - localCmdPkg = "lsst.ctrl.mpexec.cli.cmd" diff --git a/python/lsst/ctrl/mpexec/cmdLineFwk.py b/python/lsst/ctrl/mpexec/cmdLineFwk.py index 43436dbf..d3cd2416 100644 --- a/python/lsst/ctrl/mpexec/cmdLineFwk.py +++ b/python/lsst/ctrl/mpexec/cmdLineFwk.py @@ -486,29 +486,23 @@ def makePipeline(self, args: SimpleNamespace) -> Pipeline: # loop over all pipeline actions and apply them in order for action in args.pipeline_actions: if action.action == "add_instrument": - pipeline.addInstrument(action.value) elif action.action == "new_task": - pipeline.addTask(action.value, action.label) elif action.action == "delete_task": - pipeline.removeTask(action.label) elif action.action == "config": - # action value string is "field=value", split it at '=' field, _, value = action.value.partition("=") pipeline.addConfigOverride(action.label, field, value) elif action.action == "configfile": - pipeline.addConfigFile(action.label, action.value) else: - raise ValueError(f"Unexpected pipeline action: {action.action}") if args.save_pipeline: diff --git a/python/lsst/ctrl/mpexec/dotTools.py b/python/lsst/ctrl/mpexec/dotTools.py index bb46c43e..f59b39f3 100644 --- a/python/lsst/ctrl/mpexec/dotTools.py +++ b/python/lsst/ctrl/mpexec/dotTools.py @@ -167,10 +167,8 @@ def graph2dot(qgraph: QuantumGraph, file: Any) -> None: allDatasetRefs: dict[str, str] = {} for taskId, taskDef in enumerate(qgraph.taskGraph): - quanta = qgraph.getNodesForTask(taskDef) for qId, quantumNode in enumerate(quanta): - # node for a task taskNodeName = "task_{}_{}".format(taskId, qId) _renderQuantumNode(taskNodeName, taskDef, quantumNode, file) @@ -254,7 +252,6 @@ def expand_dimensions(connection: connectionTypes.BaseConnection) -> list[str]: metadataNodesToLink = set() for idx, taskDef in enumerate(sorted(pipeline, key=lambda x: x.label)): - # node for a task taskNodeName = "task{}".format(idx) diff --git a/python/lsst/ctrl/mpexec/mock_task.py b/python/lsst/ctrl/mpexec/mock_task.py index 6727129f..8eada76b 100644 --- a/python/lsst/ctrl/mpexec/mock_task.py +++ b/python/lsst/ctrl/mpexec/mock_task.py @@ -119,7 +119,6 @@ def _checkMembership(self, ref: Union[List[DatasetRef], DatasetRef], inout: set) class MockPipelineTaskConfig(PipelineTaskConfig, pipelineConnections=PipelineTaskConnections): - failCondition: Field[str] = Field( dtype=str, default="", diff --git a/python/lsst/ctrl/mpexec/mpGraphExecutor.py b/python/lsst/ctrl/mpexec/mpGraphExecutor.py index 0863d4ac..a5f70a90 100644 --- a/python/lsst/ctrl/mpexec/mpGraphExecutor.py +++ b/python/lsst/ctrl/mpexec/mpGraphExecutor.py @@ -456,7 +456,6 @@ def _executeQuantaInProcess(self, graph: QuantumGraph, butler: Butler, report: R successCount, totalCount = 0, len(graph) failedNodes: set[QuantumNode] = set() for qnode in graph: - assert qnode.quantum.dataId is not None, "Quantum DataId cannot be None" # Any failed inputs mean that the quantum has to be skipped. @@ -564,7 +563,6 @@ def _executeQuantaMP(self, graph: QuantumGraph, butler: Butler, report: Report) finishedCount, failedCount = 0, 0 while jobs.pending or jobs.running: - _LOG.debug("#pendingJobs: %s", len(jobs.pending)) _LOG.debug("#runningJobs: %s", len(jobs.running)) diff --git a/python/lsst/ctrl/mpexec/showInfo.py b/python/lsst/ctrl/mpexec/showInfo.py index 35cc4fdd..530f49c7 100644 --- a/python/lsst/ctrl/mpexec/showInfo.py +++ b/python/lsst/ctrl/mpexec/showInfo.py @@ -236,7 +236,6 @@ def _showConfigHistory(self, pipeline: Pipeline, showArgs: str) -> None: found = False for taskDef in tasks: - config = taskDef.config # Look for any matches in the config hierarchy for this name diff --git a/python/lsst/ctrl/mpexec/singleQuantumExecutor.py b/python/lsst/ctrl/mpexec/singleQuantumExecutor.py index f3330dcc..1635290a 100644 --- a/python/lsst/ctrl/mpexec/singleQuantumExecutor.py +++ b/python/lsst/ctrl/mpexec/singleQuantumExecutor.py @@ -153,7 +153,6 @@ def _execute(self, taskDef: TaskDef, quantum: Quantum, butler: Butler) -> Quantu startTime = time.time() with self.captureLogging(taskDef, quantum, butler) as captureLog: - # Save detailed resource usage before task start to metadata. quantumMetadata = _TASK_METADATA_TYPE() logInfo(None, "prep", metadata=quantumMetadata) # type: ignore diff --git a/tests/test_cmdLineFwk.py b/tests/test_cmdLineFwk.py index 71d577ec..3a4e0c98 100644 --- a/tests/test_cmdLineFwk.py +++ b/tests/test_cmdLineFwk.py @@ -360,7 +360,6 @@ def testMakeGraphFromSave(self): fwk = CmdLineFwk() with makeTmpFile(suffix=".qgraph") as tmpname, makeSQLiteRegistry() as registryConfig: - # make non-empty graph and store it in a file qgraph = _makeQGraph() with open(tmpname, "wb") as saveFile: From 09e485d890e5342f0b2a99544df056af2114810f Mon Sep 17 00:00:00 2001 From: Jim Bosch Date: Tue, 3 Jan 2023 11:43:56 -0500 Subject: [PATCH 5/6] Remove unused type: ignore. --- python/lsst/ctrl/mpexec/cli/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/ctrl/mpexec/cli/utils.py b/python/lsst/ctrl/mpexec/cli/utils.py index ddd4e56d..1b76b9b9 100644 --- a/python/lsst/ctrl/mpexec/cli/utils.py +++ b/python/lsst/ctrl/mpexec/cli/utils.py @@ -148,4 +148,4 @@ def makePipelineActions( class PipetaskCommand(MWCommand): """Command subclass with pipetask-command specific overrides.""" - extra_epilog = "See 'pipetask --help' for more options." # type: ignore + extra_epilog = "See 'pipetask --help' for more options." From e2bca952991c002452e04d9bc86cfe351da2cd92 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Tue, 28 Feb 2023 15:37:42 -0700 Subject: [PATCH 6/6] Formally declare this branch only depends on v25 dependencies --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 31faf436..845e99a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,10 +20,10 @@ classifiers = [ ] keywords = ["lsst"] dependencies = [ - "lsst-utils", - "lsst-daf-butler", - "lsst-pex-config", - "lsst-pipe-base", + "lsst-utils >=25.0,<25.100", + "lsst-daf-butler >=25.0,<25.100", + "lsst-pex-config >=25.0,<25.100", + "lsst-pipe-base >=25.0,<25.100", "click", "astropy", "pydantic",