From bfe00660347a729288096028899d544b1ebb93d3 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Mon, 9 Dec 2024 12:42:41 -0500 Subject: [PATCH 01/38] pydantic-model initial commit - pydantic model UI widget creation based on CLI model - does no processing, creates yaml files --- recOrder/plugin/gui.py | 6 +- recOrder/plugin/tab_recon.py | 495 +++++++++++++++++++++++++++++++++++ 2 files changed, 500 insertions(+), 1 deletion(-) create mode 100644 recOrder/plugin/tab_recon.py diff --git a/recOrder/plugin/gui.py b/recOrder/plugin/gui.py index 84cf37a7..04fd0f35 100644 --- a/recOrder/plugin/gui.py +++ b/recOrder/plugin/gui.py @@ -9,7 +9,7 @@ from qtpy import QtCore, QtGui, QtWidgets - +from recOrder.plugin import tab_recon class Ui_Form(object): def setupUi(self, Form): @@ -924,6 +924,10 @@ def setupUi(self, Form): self.scrollArea_4.setWidget(self.scrollAreaWidgetContents_4) self.gridLayout_6.addWidget(self.scrollArea_4, 4, 0, 1, 1) self.tabWidget.addTab(self.Acquisition, "") + + self.recon_tab = tab_recon.Ui_Form() + self.tabWidget.addTab(self.recon_tab.recon_tab_widget, 'Reconstruction') + self.Display = QtWidgets.QWidget() self.Display.setObjectName("Display") self.gridLayout_18 = QtWidgets.QGridLayout(self.Display) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py new file mode 100644 index 00000000..a822079f --- /dev/null +++ b/recOrder/plugin/tab_recon.py @@ -0,0 +1,495 @@ +import os, sys, glob, ntpath, subprocess, traceback, json, time +from pathlib import Path, PurePath +from qtpy import QtCore, QtGui +from qtpy.QtGui import QPixmap, QPainter, QCursor +from qtpy.QtCore import Qt, QTimer, QSize +from qtpy.QtWidgets import * +from magicgui.widgets import * +from napari.qt.threading import thread_worker + +import pydantic, datetime, uuid +from enum import Enum +from typing import Optional, Union +from typing import Final +from magicgui import widgets +from magicgui.type_map import get_widget_class +import warnings + +from recOrder.io import utils +from recOrder.cli import settings + +# Main class for the Reconstruction tab +# Not efficient since instantiated from GUI +# Does not have access to common functions in main_widget +# ToDo : From main_widget and pass self reference +class Ui_Form(object): + def __init__(self): + super().__init__() + self.current_dir_path = str(Path.cwd()) + self.current_save_path = str(Path.cwd()) + self.input_directory = str(Path.cwd()) + self.save_directory = str(Path.cwd()) + + # Top level parent + self.recon_tab_widget = QWidget() + self.recon_tab_layout = QVBoxLayout() + self.recon_tab_layout.setAlignment(Qt.AlignTop) + self.recon_tab_layout.setContentsMargins(0,0,0,0) + self.recon_tab_layout.setSpacing(0) + self.recon_tab_widget.setLayout(self.recon_tab_layout) + + self.recon_tab_container = widgets.Container(name='Main', scrollable=True) + self.recon_tab_layout.addWidget(self.recon_tab_container.native) + + # Top level - Selection modes, model creation and running + self.modes_widget = QWidget() + self.modes_layout = QHBoxLayout() + self.modes_layout.setAlignment(Qt.AlignTop) + self.modes_widget.setLayout(self.modes_layout) + self.modes_widget.setMaximumHeight(60) + + # For now replicate CLI processing modes - these could reside in the CLI settings file as well + # for consistency + OPTION_TO_MODEL_DICT = { + "birefringence": {"enabled":False, "setting":None}, + "phase": {"enabled":False, "setting":None}, + "fluorescence": {"enabled":False, "setting":None}, + } + self.modes_selected = OPTION_TO_MODEL_DICT + + # Make a copy of the Reconstruction settings mode, these will be used as template + for mode in self.modes_selected.keys(): + self.modes_selected[mode]["setting"] = settings.ReconstructionSettings.__fields__[mode] + + # Checkboxes for the modes to select single or combination of modes + for mode in self.modes_selected.keys(): + self.modes_selected[mode]["Checkbox"] = widgets.Checkbox( + name=mode, + label=mode + ) + self.modes_layout.addWidget(self.modes_selected[mode]["Checkbox"].native) + + # PushButton to create a copy of the model - UI + self.reconstruction_mode_enabler = widgets.PushButton( + name="CreateModel", + label="Create Model" + ) + self.reconstruction_mode_enabler.clicked.connect(self._create_acq_contols) + + # PushButton to validate and create the yaml file(s) based on selection + self.build_button = widgets.PushButton(name="Build && Run Model") + + # PushButton to clear all copies of models that are create for UI + self.reconstruction_mode_clear = widgets.PushButton( + name="ClearModels", + label="Clear All Models" + ) + self.reconstruction_mode_clear.clicked.connect(self._clear_all_models) + + # Editable List holding pydantic class(es) as per user selection + self.pydantic_classes = list() + self.index = 0 + + self.modes_layout.addWidget(self.reconstruction_mode_enabler.native) + self.modes_layout.addWidget(self.build_button.native) + self.modes_layout.addWidget(self.reconstruction_mode_clear.native) + self.recon_tab_container.native.layout().addWidget(self.modes_widget) + + # Top level - Data Input + self.modes_widget2 = QWidget() + self.modes_layout2 = QHBoxLayout() + self.modes_layout2.setAlignment(Qt.AlignTop) + self.modes_widget2.setLayout(self.modes_layout2) + + self.reconstruction_input_data_loc = widgets.LineEdit( + name="", + value=self.input_directory + ) + self.reconstruction_input_data = widgets.PushButton( + name="InputData", + label="Input Data" + ) + self.reconstruction_input_data.clicked.connect(self.browse_dir_path_input) + self.modes_layout2.addWidget(self.reconstruction_input_data_loc.native) + self.modes_layout2.addWidget(self.reconstruction_input_data.native) + self.recon_tab_container.native.layout().addWidget(self.modes_widget2) + + # Top level - Central scrollable component which will hold Editable/(vertical) Expanding UI + self.recon_tab_scrollArea_settings = QScrollArea() + self.recon_tab_scrollArea_settings.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) + self.recon_tab_scrollArea_settings.setWidgetResizable(True) + self.recon_tab_qwidget_settings = QWidget() + self.recon_tab_qwidget_settings_layout = QVBoxLayout() + self.recon_tab_qwidget_settings_layout.setSpacing(10) + self.recon_tab_qwidget_settings_layout.setAlignment(Qt.AlignTop) + self.recon_tab_qwidget_settings.setLayout(self.recon_tab_qwidget_settings_layout) + self.recon_tab_scrollArea_settings.setWidget(self.recon_tab_qwidget_settings) + self.recon_tab_layout.addWidget(self.recon_tab_scrollArea_settings) + + # Temp placeholder component to display, json pydantic output, validation msg, etc + # ToDo: Move to plugin message/error handling + self.json_display = widgets.Label(value="") + _scrollArea = QScrollArea() + _scrollArea.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) + _scrollArea.setWidgetResizable(True) + _scrollArea.setMaximumHeight(200) + _qwidget_settings = QWidget() + _qwidget_settings_layout = QVBoxLayout() + _qwidget_settings_layout.setAlignment(Qt.AlignTop) + _qwidget_settings.setLayout(_qwidget_settings_layout) + _scrollArea.setWidget(_qwidget_settings) + self.recon_tab_layout.addWidget(_scrollArea) + self.build_button.clicked.connect(self.display_json_callback) + _qwidget_settings_layout.addWidget(self.json_display.native) + + # Copied from main_widget + # ToDo: utilize common functions + # Input data selector + def browse_dir_path_input(self): + result = self._open_file_dialog(self.current_dir_path, "dir") + self.directory = result + self.current_dir_path = result + self.input_directory = result + self.reconstruction_input_data_loc.value = self.input_directory + + # Copied from main_widget + # ToDo: utilize common functions + # Output data selector + def browse_dir_path_output(self, elem): + result = self._open_file_dialog(self.current_dir_path, "dir") + self.directory = result + self.current_dir_path = result + self.save_directory = result + elem.value = self.save_directory + + # Creates UI controls from model based on selections + def _create_acq_contols(self): + # initialize the top container and specify what pydantic class to map from + pydantic_class = settings.ReconstructionSettings + + # Make a copy of selections and unsed for deletion + self.selected_modes = [] + self.selected_modes_del = [] + self.selected_modes_vals = {} + for mode in self.modes_selected.keys(): + enabled = self.modes_selected[mode]["Checkbox"].value + self.selected_modes_vals[mode] = enabled + if not enabled: + self.selected_modes_del.append(mode) + pydantic_class.__fields__[mode] = None + else: + self.selected_modes.append(mode) + pydantic_class.__fields__[mode] = self.modes_selected[mode]["setting"] + + # Container holding the pydantic UI components + # Multiple instances/copies since more than 1 might be created + recon_pydantic_container = widgets.Container(name='-and-'.join(self.selected_modes), scrollable=False) + self.add_pydantic_to_container(pydantic_class, recon_pydantic_container) + + # Add this container to the main scrollable widget + self.recon_tab_qwidget_settings_layout.addWidget(recon_pydantic_container.native) + + # Line seperator between pydantic UI components + _line = QFrame() + _line.setMinimumWidth(1) + _line.setFixedHeight(2) + _line.setFrameShape(QFrame.HLine) + _line.setFrameShadow(QFrame.Sunken) + _line.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) + _line.setStyleSheet("margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;") + + # PushButton to delete a UI container + # Use case when a wrong selection of input modes get selected eg Bire+Fl + # Preferably this root level validation should occur before values arevalidated + # in order to display and avoid this to occur + _del_button = widgets.PushButton(name="Delete this Model") + + # Final constant UI val and identifier + _idx: Final[int] = self.index + _str: Final[str] = uuid.uuid4() + + # Output Data location + # These could be multiple based on user selection for each model + # Inherits from Input by default at creation time + _output_data_loc = widgets.LineEdit( + name="", + value=self.input_directory + ) + _output_data = widgets.PushButton( + name="OutputData", + label="Output Data" + ) + + # Passing all UI components that would be deleted + _del_button.clicked.connect(lambda: self._delete_model(recon_pydantic_container.native, _output_data_loc.native, _output_data.native, _del_button.native, _line, _idx, _str)) + + # Passing location label to output location selector + _output_data.clicked.connect(lambda: self.browse_dir_path_output(_output_data_loc)) + + # HBox for Output Data + _hBox_widget = QWidget() + _hBox_layout = QHBoxLayout() + _hBox_layout.setAlignment(Qt.AlignTop) + _hBox_widget.setLayout(_hBox_layout) + _hBox_layout.addWidget(_output_data_loc.native) + _hBox_layout.addWidget(_output_data.native) + + self.recon_tab_qwidget_settings_layout.addWidget(_hBox_widget) + self.recon_tab_qwidget_settings_layout.addWidget(_del_button.native) + self.recon_tab_qwidget_settings_layout.addWidget(_line) + + # Dynamic/modifying UI probably needs this + self.recon_tab_qwidget_settings_layout.addStretch() + + # Store a copy of the pydantic container along with all its associated components and properties + # We dont needs a copy of the class but storing for now + # This will be used for making deletion edits and looping to create our final run output + # uuid - used for identiying in editable list + self.pydantic_classes.append({'uuid':_str, 'class':pydantic_class, 'input':self.reconstruction_input_data_loc, 'output':_output_data_loc, 'container':recon_pydantic_container, 'selected_modes':self.selected_modes.copy(), 'selected_modes_del':self.selected_modes_del.copy(), 'selected_modes_vals':self.selected_modes_vals.copy()}) + self.index += 1 + + # UI components deletion - maybe just needs the parent container instead of individual components + def _delete_model(self, wid1, wid2, wid3, wid4, wid5, index, _str): + if wid5 is not None: + wid5.setParent(None) + if wid4 is not None: + wid4.setParent(None) + if wid3 is not None: + wid3.setParent(None) + if wid2 is not None: + wid2.setParent(None) + if wid1 is not None: + wid1.setParent(None) + + # Find and remove the class from our pydantic model list using uuid + i=0 + for item in self.pydantic_classes: + if item["uuid"] == _str: + self.pydantic_classes.pop(i) + self.json_display.value = "" + return + i += 1 + + # Clear all the generated pydantic models and clears the pydantic model list + def _clear_all_models(self): + index = self.recon_tab_qwidget_settings_layout.count()-1 + while(index >= 0): + myWidget = self.recon_tab_qwidget_settings_layout.itemAt(index).widget() + if myWidget is not None: + myWidget.setParent(None) + index -=1 + self.pydantic_classes.clear() + self.json_display.value = "" + self.index = 0 + + # Displays the json output from the pydantic model UI selections by user + # Loops through all our stored pydantic classes + def display_json_callback(self): + # we dont want to have a partial run if there are N models + # so we will validate them all first and then run in a second loop + # first pass for validating + # second pass for creating yaml and processing + for item in self.pydantic_classes: + cls = item['class'] # not used + cls_container = item['container'] + selected_modes = item['selected_modes'] + selected_modes_del = item['selected_modes_del'] + selected_modes_vals = item['selected_modes_vals'] + + # build up the arguments for the pydantic model given the current container + cls = settings.ReconstructionSettings + + for mode in self.modes_selected.keys(): + enabled = selected_modes_vals[mode] + if not enabled: + cls.__fields__[mode] = None + else: + cls.__fields__[mode] = self.modes_selected[mode]["setting"] + + # get the kwargs from the container/class + pydantic_kwargs = {} + self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs) + + # For list element, this needs to be cleaned and parsed back as an array + pydantic_kwargs["input_channel_names"] = self.remove_chars(pydantic_kwargs["input_channel_names"], ['[',']', '\'', ' ']) + pydantic_kwargs["input_channel_names"] = pydantic_kwargs["input_channel_names"].split(',') + + # Modes that are not used needs to be purged from the class to reflect + # the same on the container side + for mode in selected_modes_del: + del cls.__fields__[mode] + + # instantiate the pydantic model form the kwargs we just pulled + # validate and return any meaning info for user + try : + pydantic_model = cls.parse_obj(pydantic_kwargs) + except pydantic.ValidationError as exc: + self.json_display.value = exc.errors()[0] + return + + # generate a json from the instantiated model, update the json_display + self.json_format = pydantic_model.json(indent=4) + self.json_display.value = self.json_format + + self.json_display.value = "" + + # generate a time-stamp for our yaml files to avoid overwriting + # files generated at the same time will have an index suffix + now = datetime.datetime.now() + unique_id = now.strftime("%Y_%m_%d_%H_%M_%S") + + i = 0 + for item in self.pydantic_classes: + i += 1 + cls = item['class'] # not used + cls_container = item['container'] + selected_modes = item['selected_modes'] + selected_modes_del = item['selected_modes_del'] + selected_modes_vals = item['selected_modes_vals'] + + # gather input/out locations + input_dir = f"{item['input'].value}" + output_dir = f"{item['output'].value}" + + # build up the arguments for the pydantic model given the current container + cls = settings.ReconstructionSettings + + for mode in self.modes_selected.keys(): + enabled = selected_modes_vals[mode] + if not enabled: + cls.__fields__[mode] = None + else: + cls.__fields__[mode] = self.modes_selected[mode]["setting"] + + pydantic_kwargs = {} + self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs) + + pydantic_kwargs["input_channel_names"] = self.remove_chars(pydantic_kwargs["input_channel_names"], ['[',']', '\'', ' ']) + pydantic_kwargs["input_channel_names"] = pydantic_kwargs["input_channel_names"].split(',') + + for mode in selected_modes_del: + del cls.__fields__[mode] + + # instantiate the pydantic model form the kwargs we just pulled + try : + pydantic_model = cls.parse_obj(pydantic_kwargs) + except pydantic.ValidationError as exc: + self.json_display.value = exc.errors()[0] + return + + # generate a json from the instantiated model, update the json_display + # most of this will end up in a table as processing proceeds + self.json_format = pydantic_model.json(indent=4) + addl_txt = "ID:" + unique_id + "-"+ str(i) + "\nInput:" + input_dir + "\nOutput:" + output_dir + self.json_display.value = self.json_display.value + addl_txt + "\n" + self.json_format+ "\n\n" + + # save the yaml files + save_config_path = str(Path.cwd()) + dir_ = save_config_path + yml_file = "-and-".join(selected_modes) + yml_file = yml_file+"-"+unique_id+"-"+str(i)+".yml" + config_path = os.path.join(dir_ ,"examples", yml_file) + utils.model_to_yaml(pydantic_model, config_path) + + # util function to parse list elements displayed as string + def remove_chars(self, string, chars_to_remove): + for char in chars_to_remove: + string = string.replace(char, '') + return string + + # Main function to add pydantic model to container + # https://github.com/chrishavlin/miscellaneous_python/blob/main/src/pydantic_magicgui_roundtrip.py + # Has limitation and can cause breakages for unhandled or incorrectly handled types + # Cannot handle Union types/typing - for now being handled explicitly + # Ignoring NoneType since those should be Optional but maybe needs displaying ?? + # ToDo: Needs revisitation, Union check + # Displaying Union field "time_indices" as LineEdit component + def add_pydantic_to_container(self, py_model: Union[pydantic.BaseModel, pydantic.main.ModelMetaclass], container: widgets.Container): + # recursively traverse a pydantic model adding widgets to a container. When a nested + # pydantic model is encountered, add a new nested container + for field, field_def in py_model.__fields__.items(): + if field_def is not None: + ftype = field_def.type_ + if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass): + # the field is a pydantic class, add a container for it and fill it + new_widget_cls = widgets.Container + new_widget = new_widget_cls(name=field_def.name) + self.add_pydantic_to_container(ftype, new_widget) + elif field == "time_indices": #ToDo: Implement Union check + new_widget_cls, ops = get_widget_class(None, str, dict(name=field, value=field_def.default)) + new_widget = new_widget_cls(**ops) + if isinstance(new_widget, widgets.EmptyWidget): + warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + else: + # parse the field, add appropriate widget + new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=field_def.default)) + new_widget = new_widget_cls(**ops) + if isinstance(new_widget, widgets.EmptyWidget): + warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + container.append(new_widget) + + # refer - add_pydantic_to_container() for comments + def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pydantic_kwargs: dict): + # given a container that was instantiated from a pydantic model, get the arguments + # needed to instantiate that pydantic model from the container. + + # traverse model fields, pull out values from container + for field, field_def in pydantic_model.__fields__.items(): + if field_def is not None: + ftype = field_def.type_ + if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass): + # go deeper + pydantic_kwargs[field] = {} # new dictionary for the new nest level + # any pydantic class will be a container, so pull that out to pass + # to the recursive call + sub_container = getattr(container, field_def.name) + self.get_pydantic_kwargs(sub_container, ftype, pydantic_kwargs[field]) + else: + # not a pydantic class, just pull the field value from the container + if hasattr(container, field_def.name): + value = getattr(container, field_def.name).value + pydantic_kwargs[field] = value + + # copied from main_widget + # file open/select dialog + def _open_file_dialog(self, default_path, type): + return self._open_dialog("select a directory", str(default_path), type) + + def _open_dialog(self, title, ref, type): + """ + opens pop-up dialogue for the user to choose a specific file or directory. + + Parameters + ---------- + title: (str) message to display at the top of the pop up + ref: (str) reference path to start the search at + type: (str) type of file the user is choosing (dir, file, or save) + + Returns + ------- + + """ + + options = QFileDialog.DontUseNativeDialog + if type == "dir": + path = QFileDialog.getExistingDirectory( + None, title, ref, options=options + ) + elif type == "file": + path = QFileDialog.getOpenFileName( + None, title, ref, options=options + )[0] + elif type == "save": + path = QFileDialog.getSaveFileName( + None, "Choose a save name", ref, options=options + )[0] + else: + raise ValueError("Did not understand file dialogue type") + + return path + +# VScode debugging +if __name__ == "__main__": + import napari + napari.Viewer() + napari.run() \ No newline at end of file From b7210637ce6f9e4f141074898b6cd72928c71ba7 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Fri, 13 Dec 2024 07:17:18 -0500 Subject: [PATCH 02/38] Updated prototype - partially working - refactored model to GUI implementation - added table to hold processing info, entries purge on completion - using click testing for now in prototyping - creates yaml to GUI - using napari notifications for msgs - testing on Windows --- recOrder/plugin/tab_recon.py | 841 +++++++++++++++++++++++++++++------ 1 file changed, 716 insertions(+), 125 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index a822079f..dd02c054 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1,15 +1,14 @@ -import os, sys, glob, ntpath, subprocess, traceback, json, time -from pathlib import Path, PurePath -from qtpy import QtCore, QtGui -from qtpy.QtGui import QPixmap, QPainter, QCursor -from qtpy.QtCore import Qt, QTimer, QSize +import os, json +from pathlib import Path + +from qtpy import QtCore + +from qtpy.QtCore import Qt from qtpy.QtWidgets import * from magicgui.widgets import * -from napari.qt.threading import thread_worker import pydantic, datetime, uuid -from enum import Enum -from typing import Optional, Union +from typing import Union, Literal from typing import Final from magicgui import widgets from magicgui.type_map import get_widget_class @@ -17,18 +16,41 @@ from recOrder.io import utils from recOrder.cli import settings +from napari.utils import notifications + +from click.testing import CliRunner +from recOrder.cli.main import cli + +from concurrent.futures import ThreadPoolExecutor + +STATUS_submitted = "Submitted" +STATUS_running = "Running" +STATUS_finished = "Finished" +STATUS_errored = "Errored" +MSG_SUCCESS = {'msg':'success'} + +# For now replicate CLI processing modes - these could reside in the CLI settings file as well +# for consistency +OPTION_TO_MODEL_DICT = { + "birefringence": {"enabled":False, "setting":None}, + "phase": {"enabled":False, "setting":None}, + "fluorescence": {"enabled":False, "setting":None}, +} + # Main class for the Reconstruction tab # Not efficient since instantiated from GUI # Does not have access to common functions in main_widget # ToDo : From main_widget and pass self reference class Ui_Form(object): + def __init__(self): super().__init__() self.current_dir_path = str(Path.cwd()) self.current_save_path = str(Path.cwd()) self.input_directory = str(Path.cwd()) self.save_directory = str(Path.cwd()) + self.yaml_model_file = str(Path.cwd()) # Top level parent self.recon_tab_widget = QWidget() @@ -37,29 +59,20 @@ def __init__(self): self.recon_tab_layout.setContentsMargins(0,0,0,0) self.recon_tab_layout.setSpacing(0) self.recon_tab_widget.setLayout(self.recon_tab_layout) - - self.recon_tab_container = widgets.Container(name='Main', scrollable=True) - self.recon_tab_layout.addWidget(self.recon_tab_container.native) # Top level - Selection modes, model creation and running self.modes_widget = QWidget() self.modes_layout = QHBoxLayout() self.modes_layout.setAlignment(Qt.AlignTop) self.modes_widget.setLayout(self.modes_layout) - self.modes_widget.setMaximumHeight(60) - - # For now replicate CLI processing modes - these could reside in the CLI settings file as well - # for consistency - OPTION_TO_MODEL_DICT = { - "birefringence": {"enabled":False, "setting":None}, - "phase": {"enabled":False, "setting":None}, - "fluorescence": {"enabled":False, "setting":None}, - } - self.modes_selected = OPTION_TO_MODEL_DICT + self.modes_widget.setMaximumHeight(50) + self.modes_widget.setMinimumHeight(50) + + self.modes_selected = OPTION_TO_MODEL_DICT.copy() # Make a copy of the Reconstruction settings mode, these will be used as template for mode in self.modes_selected.keys(): - self.modes_selected[mode]["setting"] = settings.ReconstructionSettings.__fields__[mode] + self.modes_selected[mode]["setting"] = None # Checkboxes for the modes to select single or combination of modes for mode in self.modes_selected.keys(): @@ -78,6 +91,7 @@ def __init__(self): # PushButton to validate and create the yaml file(s) based on selection self.build_button = widgets.PushButton(name="Build && Run Model") + self.build_button.clicked.connect(self.display_json_callback) # PushButton to clear all copies of models that are create for UI self.reconstruction_mode_clear = widgets.PushButton( @@ -93,26 +107,53 @@ def __init__(self): self.modes_layout.addWidget(self.reconstruction_mode_enabler.native) self.modes_layout.addWidget(self.build_button.native) self.modes_layout.addWidget(self.reconstruction_mode_clear.native) - self.recon_tab_container.native.layout().addWidget(self.modes_widget) + self.recon_tab_layout.addWidget(self.modes_widget) # Top level - Data Input self.modes_widget2 = QWidget() self.modes_layout2 = QHBoxLayout() self.modes_layout2.setAlignment(Qt.AlignTop) self.modes_widget2.setLayout(self.modes_layout2) + self.modes_widget2.setMaximumHeight(50) + self.modes_widget2.setMinimumHeight(50) self.reconstruction_input_data_loc = widgets.LineEdit( name="", value=self.input_directory ) - self.reconstruction_input_data = widgets.PushButton( + self.reconstruction_input_data_btn = widgets.PushButton( name="InputData", label="Input Data" ) - self.reconstruction_input_data.clicked.connect(self.browse_dir_path_input) + self.reconstruction_input_data_btn.clicked.connect(self.browse_dir_path_input) + self.reconstruction_input_data_loc.changed.connect(self.readAndSetInputPathOnValidation) + self.modes_layout2.addWidget(self.reconstruction_input_data_loc.native) - self.modes_layout2.addWidget(self.reconstruction_input_data.native) - self.recon_tab_container.native.layout().addWidget(self.modes_widget2) + self.modes_layout2.addWidget(self.reconstruction_input_data_btn.native) + self.recon_tab_layout.addWidget(self.modes_widget2) + + _load_model_loc = widgets.LineEdit( + name="", + value=self.input_directory + ) + _load_model_btn = widgets.PushButton( + name="LoadModel", + label="Load Model" + ) + + # Passing model location label to model location selector + _load_model_btn.clicked.connect(lambda: self.browse_dir_path_model(_load_model_loc)) + + # HBox for Loading Model + _hBox_widget_model = QWidget() + _hBox_layout_model = QHBoxLayout() + _hBox_layout_model.setAlignment(Qt.AlignTop) + _hBox_widget_model.setLayout(_hBox_layout_model) + _hBox_widget_model.setMaximumHeight(50) + _hBox_widget_model.setMinimumHeight(50) + _hBox_layout_model.addWidget(_load_model_loc.native) + _hBox_layout_model.addWidget(_load_model_btn.native) + self.recon_tab_layout.addWidget(_hBox_widget_model) # Top level - Central scrollable component which will hold Editable/(vertical) Expanding UI self.recon_tab_scrollArea_settings = QScrollArea() @@ -128,9 +169,9 @@ def __init__(self): # Temp placeholder component to display, json pydantic output, validation msg, etc # ToDo: Move to plugin message/error handling - self.json_display = widgets.Label(value="") + _scrollArea = QScrollArea() - _scrollArea.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) + # _scrollArea.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) _scrollArea.setWidgetResizable(True) _scrollArea.setMaximumHeight(200) _qwidget_settings = QWidget() @@ -139,52 +180,283 @@ def __init__(self): _qwidget_settings.setLayout(_qwidget_settings_layout) _scrollArea.setWidget(_qwidget_settings) self.recon_tab_layout.addWidget(_scrollArea) - self.build_button.clicked.connect(self.display_json_callback) - _qwidget_settings_layout.addWidget(self.json_display.native) + # Table for processing entries + self.proc_table_QFormLayout = QFormLayout() + self.proc_table_QFormLayout.setSpacing(0) + self.proc_table_QFormLayout.setContentsMargins(0,0,0,0) + _proc_table_widget = QWidget() + _proc_table_widget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) + _proc_table_widget.setLayout(self.proc_table_QFormLayout) + _qwidget_settings_layout.addWidget(_proc_table_widget) + + self.worker = MyWorker() + # Copied from main_widget # ToDo: utilize common functions # Input data selector def browse_dir_path_input(self): result = self._open_file_dialog(self.current_dir_path, "dir") + if result == '': + return self.directory = result self.current_dir_path = result self.input_directory = result self.reconstruction_input_data_loc.value = self.input_directory + # call back for input LineEdit path changed manually + def readAndSetInputPathOnValidation(self): + if self.reconstruction_input_data_loc.value is None or len(self.reconstruction_input_data_loc.value) == 0: + self.reconstruction_input_data_loc.value = self.input_directory + return + if not Path(self.reconstruction_input_data_loc.value).exists(): + self.reconstruction_input_data_loc.value = self.input_directory + return + result = self.reconstruction_input_data_loc.value + self.directory = result + self.current_dir_path = result + self.input_directory = result + # Copied from main_widget # ToDo: utilize common functions # Output data selector def browse_dir_path_output(self, elem): - result = self._open_file_dialog(self.current_dir_path, "dir") + result = self._open_file_dialog(self.current_dir_path, "save") + if result == '': + return self.directory = result - self.current_dir_path = result self.save_directory = result elem.value = self.save_directory + # call back for output LineEdit path changed manually + def readAndSetOutputPathOnValidation(self, elem): + if elem.value is None or len(elem.value) == 0: + elem.value = self.input_directory + return + result = elem.value + self.directory = result + self.save_directory = result + + # Copied from main_widget + # ToDo: utilize common functions + # Output data selector + def browse_dir_path_model(self, elem): + result = self._open_file_dialog(self.current_dir_path, "file") + if result == '': + return + self.directory = result + self.current_dir_path = result + self.yaml_model_file = result + elem.value = self.yaml_model_file + + with open(result, 'r') as yaml_in: + yaml_object = utils.yaml.safe_load(yaml_in) # yaml_object will be a list or a dict + jsonString = json.dumps(self.convert(yaml_object)) + json_out = json.loads(jsonString) + json_dict = dict(json_out) + + selected_modes = list(OPTION_TO_MODEL_DICT.copy().keys()) + exclude_modes = list(OPTION_TO_MODEL_DICT.copy().keys()) + + for k in range(len(selected_modes)-1, -1, -1): + if selected_modes[k] in json_dict.keys(): + exclude_modes.pop(k) + else: + selected_modes.pop(k) + + pruned_pydantic_class, ret_msg = self.buildModel(selected_modes) + if pruned_pydantic_class is None: + self.messageBox(ret_msg) + return + + pydantic_model, ret_msg = self.get_model_from_file(self.yaml_model_file) + if pydantic_model is None: + self.messageBox(ret_msg) + return + + pydantic_model = self._create_acq_contols2(selected_modes, exclude_modes, pydantic_model, json_dict) + if pydantic_model is None: + self.messageBox("Error - pydantic model returned None") + return + + return pydantic_model + + # passes msg to napari notifications + def messageBox(self, msg, type="exc"): + if len(msg) > 0: + try: + json_object = msg + json_txt = json_object["loc"] + " >> " + json_object["msg"] + # ToDo: format it better + except: + json_txt = str(msg) + + # show is a message box + if type == "exc": + notifications.show_error(json_txt) + else: + notifications.show_info(json_txt) + + # adds processing entry to _qwidgetTabEntry_layout as row item + # row item will be purged from table as processing finishes + # there could be 3 tabs for this processing table status + # Running, Finished, Errored + def addTableEntry(self, tableEntryID, tableEntryShortDesc, tableEntryVals, proc_params): + + _scrollAreaCollapsibleBoxDisplayWidget = widgets.Label(value=tableEntryVals) # ToDo: Replace with tablular data and Stop button + + _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() + _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBoxDisplayWidget.native) + _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(Qt.AlignTop) + + _scrollAreaCollapsibleBoxWidget = QWidget() + _scrollAreaCollapsibleBoxWidget.setLayout(_scrollAreaCollapsibleBoxWidgetLayout) + + _scrollAreaCollapsibleBox = QScrollArea() + _scrollAreaCollapsibleBox.setWidgetResizable(True) + _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) + + _collapsibleBoxWidgetLayout = QVBoxLayout() + _collapsibleBoxWidgetLayout.setContentsMargins(0,0,0,0) + _collapsibleBoxWidgetLayout.setSpacing(0) + _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) + + _collapsibleBoxWidget = CollapsibleBox(tableEntryID) # tableEntryID, tableEntryShortDesc - should update with processing status + _collapsibleBoxWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) + + _expandingTabEntryWidgetLayout = QVBoxLayout() + _expandingTabEntryWidgetLayout.addWidget(_collapsibleBoxWidget) + + _expandingTabEntryWidget = QWidget() + _expandingTabEntryWidget.toolTip = tableEntryShortDesc + _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) + _expandingTabEntryWidget.layout().setContentsMargins(0,0,0,0) + _expandingTabEntryWidget.layout().setSpacing(0) + _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + + self.proc_table_QFormLayout.addRow(_expandingTabEntryWidget) + + proc_params["table_layout"] = self.proc_table_QFormLayout + proc_params["table_entry"] = _expandingTabEntryWidget + + self.worker.runInPool(proc_params) + # result = self.worker.getResult(proc_params["exp_id"]) + # print(result) + + + # Builds the model as required + def buildModel(self, selected_modes): + try: + b = None + p = None + f = None + chNames = ['State0'] + exclude_modes = ["birefringence", "phase", "fluorescence"] + if "birefringence" in selected_modes and "phase" in selected_modes: + b = settings.BirefringenceSettings() + p = settings.PhaseSettings() + chNames = ['State0','State1','State2','State3'] + exclude_modes = ["fluorescence"] + elif "birefringence" in selected_modes: + b = settings.BirefringenceSettings() + chNames = ['State0','State1','State2','State3'] + exclude_modes = ["fluorescence", "phase"] + elif "phase" in selected_modes: + p = settings.PhaseSettings() + exclude_modes = ["birefringence", "fluorescence"] + elif "fluorescence" in selected_modes: + f = settings.FluorescenceSettings() + exclude_modes = ["birefringence", "phase"] + + model = settings.ReconstructionSettings(input_channel_names=chNames, birefringence=b, phase=p, fluorescence=f) + model = self._fix_model(model, exclude_modes, 'input_channel_names', chNames) + + return model, "+".join(selected_modes) + ": MSG_SUCCESS" + except pydantic.ValidationError as exc: + return None, "+".join(selected_modes) + str(exc.errors()[0]) + + # ToDo: Temporary fix to over ride the 'input_channel_names' default value + # Needs revisitation + def _fix_model(self, model, exclude_modes, attr_key, attr_val): + try: + for mode in exclude_modes: + model = settings.ReconstructionSettings.copy(model, exclude={mode}, deep=True, update={attr_key:attr_val}) + settings.ReconstructionSettings.__setattr__(model, attr_key, attr_val) + if hasattr(model, attr_key): + model.__fields__[attr_key].default = attr_val + model.__fields__[attr_key].field_info.default = attr_val + except Exception as exc: + return print(exc.args) + return model + # Creates UI controls from model based on selections def _create_acq_contols(self): - # initialize the top container and specify what pydantic class to map from - pydantic_class = settings.ReconstructionSettings # Make a copy of selections and unsed for deletion - self.selected_modes = [] - self.selected_modes_del = [] - self.selected_modes_vals = {} + selected_modes = [] + exclude_modes = [] + for mode in self.modes_selected.keys(): enabled = self.modes_selected[mode]["Checkbox"].value - self.selected_modes_vals[mode] = enabled if not enabled: - self.selected_modes_del.append(mode) - pydantic_class.__fields__[mode] = None + exclude_modes.append(mode) else: - self.selected_modes.append(mode) - pydantic_class.__fields__[mode] = self.modes_selected[mode]["setting"] + selected_modes.append(mode) + + self._create_acq_contols2(selected_modes, exclude_modes) + + def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None, json_dict=None): + + # initialize the top container and specify what pydantic class to map from + if myLoadedModel is not None: + pydantic_class = myLoadedModel + else: + pydantic_class, ret_msg = self.buildModel(selected_modes) + if pydantic_class is None: + self.messageBox(ret_msg) + return # Container holding the pydantic UI components # Multiple instances/copies since more than 1 might be created - recon_pydantic_container = widgets.Container(name='-and-'.join(self.selected_modes), scrollable=False) - self.add_pydantic_to_container(pydantic_class, recon_pydantic_container) + recon_pydantic_container = widgets.Container(name="", scrollable=False) + + self.add_pydantic_to_container(pydantic_class, recon_pydantic_container, exclude_modes, json_dict) + + # Run a validation check to see if the selected options are permitted + # before we create the GUI + # get the kwargs from the container/class + pydantic_kwargs = {} + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args(recon_pydantic_container, pydantic_class, pydantic_kwargs, exclude_modes) + if pydantic_kwargs is None: + self.messageBox(ret_msg) + return + + # For list element, this needs to be cleaned and parsed back as an array + input_channel_names, ret_msg = self.clean_string_for_list("input_channel_names", pydantic_kwargs["input_channel_names"]) + if input_channel_names is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["input_channel_names"] = input_channel_names + + time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) + if time_indices is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["time_indices"] = time_indices + + # validate and return errors if None + pydantic_model, ret_msg = self.validate_pydantic_model(pydantic_class, pydantic_kwargs) + if pydantic_model is None: + self.messageBox(ret_msg) + return + + # generate a json from the instantiated model, update the json_display + # most of this will end up in a table as processing proceeds + json_txt, ret_msg = self.validate_and_return_json(pydantic_model) + if json_txt is None: + self.messageBox(ret_msg) + return # Add this container to the main scrollable widget self.recon_tab_qwidget_settings_layout.addWidget(recon_pydantic_container.native) @@ -215,24 +487,25 @@ def _create_acq_contols(self): name="", value=self.input_directory ) - _output_data = widgets.PushButton( + _output_data_btn = widgets.PushButton( name="OutputData", label="Output Data" ) - - # Passing all UI components that would be deleted - _del_button.clicked.connect(lambda: self._delete_model(recon_pydantic_container.native, _output_data_loc.native, _output_data.native, _del_button.native, _line, _idx, _str)) # Passing location label to output location selector - _output_data.clicked.connect(lambda: self.browse_dir_path_output(_output_data_loc)) + _output_data_btn.clicked.connect(lambda: self.browse_dir_path_output(_output_data_loc)) + _output_data_loc.changed.connect(lambda: self.readAndSetOutputPathOnValidation(_output_data_loc)) + # Passing all UI components that would be deleted + _del_button.clicked.connect(lambda: self._delete_model(recon_pydantic_container.native, _output_data_loc.native, _output_data_btn.native, _del_button.native, _line, _idx, _str)) + # HBox for Output Data _hBox_widget = QWidget() _hBox_layout = QHBoxLayout() _hBox_layout.setAlignment(Qt.AlignTop) _hBox_widget.setLayout(_hBox_layout) _hBox_layout.addWidget(_output_data_loc.native) - _hBox_layout.addWidget(_output_data.native) + _hBox_layout.addWidget(_output_data_btn.native) self.recon_tab_qwidget_settings_layout.addWidget(_hBox_widget) self.recon_tab_qwidget_settings_layout.addWidget(_del_button.native) @@ -245,11 +518,13 @@ def _create_acq_contols(self): # We dont needs a copy of the class but storing for now # This will be used for making deletion edits and looping to create our final run output # uuid - used for identiying in editable list - self.pydantic_classes.append({'uuid':_str, 'class':pydantic_class, 'input':self.reconstruction_input_data_loc, 'output':_output_data_loc, 'container':recon_pydantic_container, 'selected_modes':self.selected_modes.copy(), 'selected_modes_del':self.selected_modes_del.copy(), 'selected_modes_vals':self.selected_modes_vals.copy()}) + self.pydantic_classes.append({'uuid':_str, 'class':pydantic_class, 'input':self.reconstruction_input_data_loc, 'output':_output_data_loc, 'container':recon_pydantic_container, 'selected_modes':selected_modes.copy(), 'exclude_modes':exclude_modes.copy()}) self.index += 1 + return pydantic_model # UI components deletion - maybe just needs the parent container instead of individual components def _delete_model(self, wid1, wid2, wid3, wid4, wid5, index, _str): + if wid5 is not None: wid5.setParent(None) if wid4 is not None: @@ -266,7 +541,6 @@ def _delete_model(self, wid1, wid2, wid3, wid4, wid5, index, _str): for item in self.pydantic_classes: if item["uuid"] == _str: self.pydantic_classes.pop(i) - self.json_display.value = "" return i += 1 @@ -279,7 +553,6 @@ def _clear_all_models(self): myWidget.setParent(None) index -=1 self.pydantic_classes.clear() - self.json_display.value = "" self.index = 0 # Displays the json output from the pydantic model UI selections by user @@ -290,48 +563,48 @@ def display_json_callback(self): # first pass for validating # second pass for creating yaml and processing for item in self.pydantic_classes: - cls = item['class'] # not used + cls = item['class'] cls_container = item['container'] selected_modes = item['selected_modes'] - selected_modes_del = item['selected_modes_del'] - selected_modes_vals = item['selected_modes_vals'] + exclude_modes = item['exclude_modes'] # build up the arguments for the pydantic model given the current container - cls = settings.ReconstructionSettings - - for mode in self.modes_selected.keys(): - enabled = selected_modes_vals[mode] - if not enabled: - cls.__fields__[mode] = None - else: - cls.__fields__[mode] = self.modes_selected[mode]["setting"] + if cls is None: + self.messageBox(ret_msg) + return # get the kwargs from the container/class pydantic_kwargs = {} - self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs) + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args(cls_container, cls, pydantic_kwargs, exclude_modes) + if pydantic_kwargs is None: + self.messageBox(ret_msg) + return # For list element, this needs to be cleaned and parsed back as an array - pydantic_kwargs["input_channel_names"] = self.remove_chars(pydantic_kwargs["input_channel_names"], ['[',']', '\'', ' ']) - pydantic_kwargs["input_channel_names"] = pydantic_kwargs["input_channel_names"].split(',') - - # Modes that are not used needs to be purged from the class to reflect - # the same on the container side - for mode in selected_modes_del: - del cls.__fields__[mode] - - # instantiate the pydantic model form the kwargs we just pulled - # validate and return any meaning info for user - try : - pydantic_model = cls.parse_obj(pydantic_kwargs) - except pydantic.ValidationError as exc: - self.json_display.value = exc.errors()[0] + input_channel_names, ret_msg = self.clean_string_for_list("input_channel_names", pydantic_kwargs["input_channel_names"]) + if input_channel_names is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["input_channel_names"] = input_channel_names + + time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) + if time_indices is None: + self.messageBox(ret_msg) return + pydantic_kwargs["time_indices"] = time_indices - # generate a json from the instantiated model, update the json_display - self.json_format = pydantic_model.json(indent=4) - self.json_display.value = self.json_format + # validate and return errors if None + pydantic_model, ret_msg = self.validate_pydantic_model(cls, pydantic_kwargs) + if pydantic_model is None: + self.messageBox(ret_msg) + return - self.json_display.value = "" + # generate a json from the instantiated model, update the json_display + # most of this will end up in a table as processing proceeds + json_txt, ret_msg = self.validate_and_return_json(pydantic_model) + if json_txt is None: + self.messageBox(ret_msg) + return # generate a time-stamp for our yaml files to avoid overwriting # files generated at the same time will have an index suffix @@ -341,61 +614,189 @@ def display_json_callback(self): i = 0 for item in self.pydantic_classes: i += 1 - cls = item['class'] # not used + cls = item['class'] cls_container = item['container'] selected_modes = item['selected_modes'] - selected_modes_del = item['selected_modes_del'] - selected_modes_vals = item['selected_modes_vals'] + exclude_modes = item['exclude_modes'] # gather input/out locations input_dir = f"{item['input'].value}" output_dir = f"{item['output'].value}" # build up the arguments for the pydantic model given the current container - cls = settings.ReconstructionSettings - - for mode in self.modes_selected.keys(): - enabled = selected_modes_vals[mode] - if not enabled: - cls.__fields__[mode] = None - else: - cls.__fields__[mode] = self.modes_selected[mode]["setting"] + if cls is None: + self.messageBox(ret_msg) + return pydantic_kwargs = {} - self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs) + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args(cls_container, cls, pydantic_kwargs, exclude_modes) + if pydantic_kwargs is None: + self.messageBox(ret_msg) + return - pydantic_kwargs["input_channel_names"] = self.remove_chars(pydantic_kwargs["input_channel_names"], ['[',']', '\'', ' ']) - pydantic_kwargs["input_channel_names"] = pydantic_kwargs["input_channel_names"].split(',') + input_channel_names, ret_msg = self.clean_string_for_list("input_channel_names", pydantic_kwargs["input_channel_names"]) + if input_channel_names is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["input_channel_names"] = input_channel_names - for mode in selected_modes_del: - del cls.__fields__[mode] + time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) + if time_indices is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["time_indices"] = time_indices - # instantiate the pydantic model form the kwargs we just pulled - try : - pydantic_model = cls.parse_obj(pydantic_kwargs) - except pydantic.ValidationError as exc: - self.json_display.value = exc.errors()[0] + # validate and return errors if None + pydantic_model, ret_msg = self.validate_pydantic_model(cls, pydantic_kwargs) + if pydantic_model is None: + self.messageBox(ret_msg) return # generate a json from the instantiated model, update the json_display # most of this will end up in a table as processing proceeds - self.json_format = pydantic_model.json(indent=4) - addl_txt = "ID:" + unique_id + "-"+ str(i) + "\nInput:" + input_dir + "\nOutput:" + output_dir - self.json_display.value = self.json_display.value + addl_txt + "\n" + self.json_format+ "\n\n" - + json_txt, ret_msg = self.validate_and_return_json(pydantic_model) + if json_txt is None: + self.messageBox(ret_msg) + return + # save the yaml files + # ToDo: error catching and validation for path + # path selection ??? save_config_path = str(Path.cwd()) dir_ = save_config_path - yml_file = "-and-".join(selected_modes) - yml_file = yml_file+"-"+unique_id+"-"+str(i)+".yml" + yml_file_name = "-and-".join(selected_modes) + yml_file = yml_file_name+"-"+unique_id+"-"+str(i)+".yml" config_path = os.path.join(dir_ ,"examples", yml_file) utils.model_to_yaml(pydantic_model, config_path) - + + # Input params for table entry + # Once ALL entries are entered we can deleted ALL model containers + # Table will need a low priority update thread to refresh status queried from CLI + # Table entries will be purged on completion when Result is returned OK + # Table entries will show an error msg when processing finishes but Result not OK + # Table fields ID / DateTime, Reconstruction type, Input Location, Output Location, Progress indicator, Stop button + + # addl_txt = "ID:" + unique_id + "-"+ str(i) + "\nInput:" + input_dir + "\nOutput:" + output_dir + # self.json_display.value = self.json_display.value + addl_txt + "\n" + json_txt+ "\n\n" + expID = "{tID}-{idx}".format(tID = unique_id, idx = i) + tableID = "{tName}: ({tID}-{idx})".format(tName = yml_file_name, tID = unique_id, idx = i) + tableDescToolTip = "{tName}: ({tID}-{idx})".format(tName = yml_file_name, tID = unique_id, idx = i) + + proc_params = {} + proc_params["exp_id"] = expID + proc_params["config_path"] = str(Path(config_path).absolute()) + proc_params["input_path"] = str(Path(input_dir).absolute()) + proc_params["output_path"] = str(Path(output_dir).absolute()) + + self.addTableEntry(tableID, tableDescToolTip, json_txt, proc_params) + # util function to parse list elements displayed as string def remove_chars(self, string, chars_to_remove): for char in chars_to_remove: string = string.replace(char, '') - return string + return string + + # util function to parse list elements displayed as string + def clean_string_for_list(self, field, string): + chars_to_remove = ['[',']', '\'', '"', ' '] + if isinstance(string, str): + string = self.remove_chars(string, chars_to_remove) + if len(string) == 0: + return None, {'msg':field + ' is invalid'} + if ',' in string: + string = string.split(',') + return string, MSG_SUCCESS + if isinstance(string, str): + string = [string] + return string, MSG_SUCCESS + return string, MSG_SUCCESS + + # util function to parse list elements displayed as string, int, int as list of strings, int range + # [1,2,3], 4,5,6 , 5-95 + def clean_string_int_for_list(self, field, string): + chars_to_remove = ['[',']', '\'', '"', ' '] + if isinstance(string, str): + string = self.remove_chars(string, chars_to_remove) + if len(string) == 0: + return None, {'msg':field + ' is invalid'} + if 'all' in string: + if Literal[string] == Literal["all"]: + return string, MSG_SUCCESS + else: + return None, {'msg':field + ' can only contain \'all\' as string field'} + if '-' in string: + string = string.split('-') + if len(string) == 2: + try: + x = int(string[0]) + if not isinstance(x, int): + raise + except Exception as exc: + return None, {'msg':field + ' first range element is not an integer'} + try: + y = int(string[1]) + if not isinstance(y, int): + raise + except Exception as exc: + return None, {'msg':field + ' second range element is not an integer'} + if y > x: + return list(range(x, y+1)), MSG_SUCCESS + else: + return None, {'msg':field + ' second integer cannot be smaller than first'} + else: + return None, {'msg':field + ' is invalid'} + if ',' in string: + string = string.split(',') + return string, MSG_SUCCESS + return string, MSG_SUCCESS + + # get the pydantic_kwargs and catches any errors in doing so + def get_and_validate_pydantic_args(self, cls_container, cls, pydantic_kwargs, exclude_modes): + try: + self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs, exclude_modes) + return pydantic_kwargs, MSG_SUCCESS + except pydantic.ValidationError as exc: + return None, exc.errors()[0] + + # validate the model and return errors for user actioning + def validate_pydantic_model(self, cls, pydantic_kwargs): + # instantiate the pydantic model form the kwargs we just pulled + try : + pydantic_model = settings.ReconstructionSettings.parse_obj(pydantic_kwargs) + return pydantic_model, MSG_SUCCESS + except pydantic.ValidationError as exc: + return None, exc.errors()[0] + + # test to make sure model coverts to json which should ensure compatibility with yaml export + def validate_and_return_json(self, pydantic_model): + try : + json_format = pydantic_model.json(indent=4) + return json_format, MSG_SUCCESS + except Exception as exc: + return None, exc.args + + # gets a copy of the model from a yaml file + # will get all fields (even those that are optional and not in yaml) and default values + # model needs further parsing against yaml file for fields + def get_model_from_file(self, model_file_path): + try : + pydantic_model = utils.yaml_to_model(model_file_path, settings.ReconstructionSettings) + if pydantic_model is None: + raise Exception("yaml_to_model - returned a None model") + return pydantic_model, MSG_SUCCESS + except Exception as exc: + return None, exc.args + + # handles json with boolean properly and converts to lowercase string + # as required + def convert(self, obj): + if isinstance(obj, bool): + return str(obj).lower() + if isinstance(obj, (list, tuple)): + return [self.convert(item) for item in obj] + if isinstance(obj, dict): + return {self.convert(key):self.convert(value) for key, value in obj.items()} + return obj # Main function to add pydantic model to container # https://github.com/chrishavlin/miscellaneous_python/blob/main/src/pydantic_magicgui_roundtrip.py @@ -404,38 +805,65 @@ def remove_chars(self, string, chars_to_remove): # Ignoring NoneType since those should be Optional but maybe needs displaying ?? # ToDo: Needs revisitation, Union check # Displaying Union field "time_indices" as LineEdit component - def add_pydantic_to_container(self, py_model: Union[pydantic.BaseModel, pydantic.main.ModelMetaclass], container: widgets.Container): + # excludes handles fields that are not supposed to show up from __fields__ + # json_dict adds ability to provide new set of default values at time of container creation + + def add_pydantic_to_container(self, py_model:Union[pydantic.BaseModel, pydantic.main.ModelMetaclass], container: widgets.Container, excludes=[], json_dict=None): # recursively traverse a pydantic model adding widgets to a container. When a nested # pydantic model is encountered, add a new nested container - for field, field_def in py_model.__fields__.items(): - if field_def is not None: - ftype = field_def.type_ + + for field, field_def in py_model.__fields__.items(): + if field_def is not None and field not in excludes: + def_val = field_def.default + ftype = field_def.type_ if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass): + json_val = None + if json_dict is not None: + json_val = json_dict[field] # the field is a pydantic class, add a container for it and fill it new_widget_cls = widgets.Container new_widget = new_widget_cls(name=field_def.name) - self.add_pydantic_to_container(ftype, new_widget) - elif field == "time_indices": #ToDo: Implement Union check - new_widget_cls, ops = get_widget_class(None, str, dict(name=field, value=field_def.default)) + self.add_pydantic_to_container(ftype, new_widget, excludes, json_val) + #ToDo: Implement Union check, tried: + # pydantic.typing.is_union(ftype) + # isinstance(ftype, types.UnionType) + # https://stackoverflow.com/questions/45957615/how-to-check-a-variable-against-union-type-during-runtime + elif isinstance(def_val, str): #field == "time_indices": + new_widget_cls, ops = get_widget_class(None, str, dict(name=field, value=def_val)) new_widget = new_widget_cls(**ops) if isinstance(new_widget, widgets.EmptyWidget): warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + elif isinstance(def_val, float): + # parse the field, add appropriate widget + if def_val > -1 and def_val < 1: + new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val, step=float(0.001))) + new_widget = new_widget_cls(**ops) + else: + new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val)) + new_widget = new_widget_cls(**ops) + if isinstance(new_widget, widgets.EmptyWidget): + warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") else: # parse the field, add appropriate widget - new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=field_def.default)) + new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val)) new_widget = new_widget_cls(**ops) if isinstance(new_widget, widgets.EmptyWidget): warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + if json_dict is not None and not isinstance(new_widget, widgets.Container): + if isinstance(new_widget, widgets.CheckBox): + new_widget.value = True if json_dict[field]=="true" else False + else: + new_widget.value = json_dict[field] container.append(new_widget) # refer - add_pydantic_to_container() for comments - def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pydantic_kwargs: dict): + def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pydantic_kwargs: dict, excludes=[], json_dict=None): # given a container that was instantiated from a pydantic model, get the arguments # needed to instantiate that pydantic model from the container. # traverse model fields, pull out values from container for field, field_def in pydantic_model.__fields__.items(): - if field_def is not None: + if field_def is not None and field not in excludes: ftype = field_def.type_ if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass): # go deeper @@ -443,7 +871,7 @@ def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pyda # any pydantic class will be a container, so pull that out to pass # to the recursive call sub_container = getattr(container, field_def.name) - self.get_pydantic_kwargs(sub_container, ftype, pydantic_kwargs[field]) + self.get_pydantic_kwargs(sub_container, ftype, pydantic_kwargs[field], excludes, json_dict) else: # not a pydantic class, just pull the field value from the container if hasattr(container, field_def.name): @@ -453,7 +881,14 @@ def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pyda # copied from main_widget # file open/select dialog def _open_file_dialog(self, default_path, type): - return self._open_dialog("select a directory", str(default_path), type) + if type == "dir": + return self._open_dialog("select a directory", str(default_path), type) + elif type == "file": + return self._open_dialog("select a file", str(default_path), type) + elif type == "save": + return self._open_dialog("save a file", str(default_path), type) + else: + return self._open_dialog("select a directory", str(default_path), type) def _open_dialog(self, title, ref, type): """ @@ -487,6 +922,162 @@ def _open_dialog(self, title, ref, type): raise ValueError("Did not understand file dialogue type") return path + +class CollapsibleBox(QWidget): + def __init__(self, title="", parent=None): + super(CollapsibleBox, self).__init__(parent) + + self.toggle_button = QToolButton( + text=title, checkable=True, checked=False + ) + self.toggle_button.setStyleSheet("QToolButton { border: none; }") + self.toggle_button.setToolButtonStyle( + QtCore.Qt.ToolButtonTextBesideIcon + ) + self.toggle_button.setArrowType(QtCore.Qt.RightArrow) + self.toggle_button.pressed.connect(self.on_pressed) + + self.toggle_animation = QtCore.QParallelAnimationGroup(self) + + self.content_area = QScrollArea(maximumHeight=0, minimumHeight=0) + self.content_area.setSizePolicy( + QSizePolicy.Expanding, QSizePolicy.Fixed + ) + self.content_area.setFrameShape(QFrame.NoFrame) + + lay = QVBoxLayout(self) + lay.setSpacing(0) + lay.setContentsMargins(0, 0, 0, 0) + lay.addWidget(self.toggle_button) + lay.addWidget(self.content_area) + + self.toggle_animation.addAnimation( + QtCore.QPropertyAnimation(self, b"minimumHeight") + ) + self.toggle_animation.addAnimation( + QtCore.QPropertyAnimation(self, b"maximumHeight") + ) + self.toggle_animation.addAnimation( + QtCore.QPropertyAnimation(self.content_area, b"maximumHeight") + ) + + # @QtCore.pyqtSlot() + def on_pressed(self): + checked = self.toggle_button.isChecked() + self.toggle_button.setArrowType( + QtCore.Qt.DownArrow if not checked else QtCore.Qt.RightArrow + ) + self.toggle_animation.setDirection( + QtCore.QAbstractAnimation.Forward + if not checked + else QtCore.QAbstractAnimation.Backward + ) + self.toggle_animation.start() + + def setContentLayout(self, layout): + lay = self.content_area.layout() + del lay + self.content_area.setLayout(layout) + collapsed_height = ( + self.sizeHint().height() - self.content_area.maximumHeight() + ) + content_height = layout.sizeHint().height() + for i in range(self.toggle_animation.animationCount()): + animation = self.toggle_animation.animationAt(i) + animation.setDuration(500) + animation.setStartValue(collapsed_height) + animation.setEndValue(collapsed_height + content_height) + + content_animation = self.toggle_animation.animationAt( + self.toggle_animation.animationCount() - 1 + ) + content_animation.setDuration(500) + content_animation.setStartValue(0) + content_animation.setEndValue(content_height) + +class MyWorker(object): + + def __init__(self): + super().__init__() + self.max_cores = os.cpu_count() + # In the case of CLI, we just need to submit requests in a non-blocking way + self.threadPool = int(self.max_cores/2) + self.results = {} + self.pool = None + # https://click.palletsprojects.com/en/stable/testing/ + self.runner = CliRunner() + self.startPool() + + def getMaxCPU_cores(self): + return self.max_cores + + def setPoolThreads(self, t): + if t > 0 and t < self.max_cores: + self.threadPool = t + + def startPool(self): + if self.pool is None: + self.pool = ThreadPoolExecutor(max_workers=self.threadPool) + + def shutDownPool(self): + self.pool.shutdown(wait=False) + + def runInPool(self, params): + self.results[params["exp_id"]] = params + self.results[params["exp_id"]]["status"] = STATUS_submitted + self.results[params["exp_id"]]["error"] = "" + self.pool.submit(self.run, params) + + def runMultiInPool(self, multi_params_as_list): + for params in multi_params_as_list: + self.results[params["exp_id"]] = params + self.results[params["exp_id"]]["status"] = STATUS_submitted + self.results[params["exp_id"]]["error"] = "" + self.pool.map(self.run, multi_params_as_list) + + def getResults(self): + return self.results + + def getResult(self, exp_id): + return self.results[exp_id] + + def run(self, params): + # thread where work is passed to CLI which will handle the + # multi-processing aspects based on resources + if params["exp_id"] not in self.results.keys(): + self.results[params["exp_id"]] = params + self.results[params["exp_id"]]["error"] = "" + + self.results[params["exp_id"]]["status"] = STATUS_running + try: + input_path = params["input_path"] + config_path = params["config_path"] + output_path = params["output_path"] + + # ToDo: replace with command line ver + result = self.runner.invoke( + cli, + [ + "reconstruct", + "-i", + str(input_path), + "-c", + str(config_path), + "-o", + str(output_path), + ], + catch_exceptions=False, + ) + + self.results[params["exp_id"]]["result"] = result + self.results[params["exp_id"]]["status"] = STATUS_finished + + _proc_table_QFormLayout = self.results[params["exp_id"]]["table_layout"] + _expandingTabEntryWidget = self.results[params["exp_id"]]["table_entry"] + _proc_table_QFormLayout.removeRow(_expandingTabEntryWidget) + except Exception as exc: + self.results[params["exp_id"]]["status"] = STATUS_errored + self.results[params["exp_id"]]["error"] = exc.args # VScode debugging if __name__ == "__main__": From f91b4e9f0013a94651eee84adcd291c08e1c7cd2 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Mon, 16 Dec 2024 15:43:39 -0500 Subject: [PATCH 03/38] first working alpha version - Multi processing using pydantic models - Implemented a client/server approach for managing jobs queue - Updates each job status based on logs - Implemented Unique ID in CLI to associate Jobs with Process - Tested on Windows with locally modified submitit --- .../cli/apply_inverse_transfer_function.py | 25 +- recOrder/cli/jobs_mgmt.py | 99 +++ recOrder/cli/main.py | 9 +- recOrder/cli/parsing.py | 13 + recOrder/cli/reconstruct.py | 4 + recOrder/io/utils.py | 4 +- recOrder/plugin/gui.py | 2 +- recOrder/plugin/tab_recon.py | 675 ++++++++++++++---- 8 files changed, 677 insertions(+), 154 deletions(-) create mode 100644 recOrder/cli/jobs_mgmt.py diff --git a/recOrder/cli/apply_inverse_transfer_function.py b/recOrder/cli/apply_inverse_transfer_function.py index b4f97a38..2f98b246 100644 --- a/recOrder/cli/apply_inverse_transfer_function.py +++ b/recOrder/cli/apply_inverse_transfer_function.py @@ -10,6 +10,9 @@ import submitit from iohub import open_ome_zarr +from typing import Final +from recOrder.cli import jobs_mgmt + from recOrder.cli import apply_inverse_models from recOrder.cli.parsing import ( config_filepath, @@ -18,6 +21,7 @@ processes_option, transfer_function_dirpath, ram_multiplier, + unique_id, ) from recOrder.cli.printing import echo_headline, echo_settings from recOrder.cli.settings import ReconstructionSettings @@ -28,6 +32,7 @@ from recOrder.io import utils from recOrder.cli.monitor import monitor_jobs +JM = jobs_mgmt.JobsManagement() def _check_background_consistency( background_shape, data_shape, input_channel_names @@ -293,6 +298,7 @@ def apply_inverse_transfer_function_cli( output_dirpath: Path, num_processes: int = 1, ram_multiplier: float = 1.0, + unique_id: str = "" ) -> None: output_metadata = get_reconstruction_output_metadata( input_position_dirpaths[0], config_filepath @@ -346,12 +352,11 @@ def apply_inverse_transfer_function_cli( slurm_partition="cpu", # more slurm_*** resource parameters here ) - + jobs = [] with executor.batch(): - for input_position_dirpath in input_position_dirpaths: - jobs.append( - executor.submit( + for input_position_dirpath in input_position_dirpaths: + job: Final = executor.submit( apply_inverse_transfer_function_single_position, input_position_dirpath, transfer_function_dirpath, @@ -359,12 +364,20 @@ def apply_inverse_transfer_function_cli( output_dirpath / Path(*input_position_dirpath.parts[-3:]), num_processes, output_metadata["channel_names"], - ) - ) + ) + jobs.append(job) echo_headline( f"{num_jobs} job{'s' if num_jobs > 1 else ''} submitted {'locally' if executor.cluster == 'local' else 'via ' + executor.cluster}." ) + if unique_id != "": # no unique_id means no job submission info being listened to + JM.startClient() + for j in jobs: + job : submitit.Job = j + job_idx : str = job.job_id + JM.putJobInListClient(unique_id, job_idx) + JM.stopClient() + monitor_jobs(jobs, input_position_dirpaths) diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py new file mode 100644 index 00000000..d8a1e628 --- /dev/null +++ b/recOrder/cli/jobs_mgmt.py @@ -0,0 +1,99 @@ +import submitit, os, json, time +import socket +from pathlib import Path +from tempfile import TemporaryDirectory + +# Jobs query object +# Todo: Not sure where these should functions should reside - ask Talon + +# def modify_dict(shared_var_jobs, k, v, lock): +# with lock: +# for key in shared_var_jobs.keys(): +# print(key) +# tmp = shared_var_jobs[key] +# tmp["count"] += 1 +# shared_var_jobs[key] = tmp +# shared_var_jobs[k] = {"count":0, "val": v} +# print(shared_var_jobs) + +SERVER_PORT = 8089 + +class JobsManagement(): + + def __init__(self, *args, **kwargs): + # self.m = Manager() + # self.shared_var_jobs = self.m.dict() + # self.lock = self.m.Lock() + self.shared_var_jobs = dict() + self.executor = submitit.AutoExecutor(folder="logs") + self.logsPath = self.executor.folder + self.tmp_path = TemporaryDirectory() + self.tempDir = os.path.join(Path(self.tmp_path.name), "tempfiles") + + def checkForJobIDFile(self, jobID, extension="out"): + files = os.listdir(self.logsPath) + try: + for file in files: + if file.endswith(extension): + if jobID in file: + file_path = os.path.join(self.logsPath, file) + f = open(file_path, "r") + txt = f.read() + f.close() + return txt + except Exception as exc: + print(exc.args) + return "" + + def startClient(self): + try: + self.clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.clientsocket.settimeout(300) + self.clientsocket.connect(('localhost', SERVER_PORT)) + self.clientsocket.settimeout(None) + except Exception as exc: + print(exc.args) + + def stopClient(self): + try: + self.clientsocket.close() + except Exception as exc: + print(exc.args) + + def putJobInListClient(self, uid: str, job: str): + try: + json_obj = {uid:job} + json_str = json.dumps(json_obj) + self.clientsocket.send(bytes(json_str, 'UTF-8')) + # p1 = Process(target=modify_dict, args=(self.shared_var_jobs, uid, job, self.lock)) + # p1.start() + # p1.join() + # p2 = Process(target=increment, args=(self.shared_var_jobs, self.lock)) + # p2.start() + # p2.join() + except Exception as exc: + print(exc.args) + + def hasSubmittedJob(self, uuid_str: str)->bool: + if uuid_str in self.shared_var_jobs.keys(): + return True + return False + + ####### below - not being used ######### + + def getJobs(self): + return self.shared_var_jobs + + def getJob(self, uuid_str: str)->submitit.Job: + if uuid_str in self.shared_var_jobs.keys(): + return self.shared_var_jobs[uuid_str] + + def cancelJob(self, uuid_str: str): + if uuid_str in self.shared_var_jobs.keys(): + job: submitit.Job = self.shared_var_jobs[uuid_str] + job.cancel() + + def getInfoOnJob(self, uuid_str: str)->list: + if uuid_str in self.shared_var_jobs.keys(): + job: submitit.Job = self.shared_var_jobs[uuid_str] + return job.results() diff --git a/recOrder/cli/main.py b/recOrder/cli/main.py index a05fb452..f4c51382 100644 --- a/recOrder/cli/main.py +++ b/recOrder/cli/main.py @@ -1,11 +1,14 @@ -import click +import click, os from recOrder.cli.apply_inverse_transfer_function import apply_inv_tf from recOrder.cli.compute_transfer_function import compute_tf from recOrder.cli.reconstruct import reconstruct + CONTEXT = {"help_option_names": ["-h", "--help"]} +DIR_PATH = os.path.dirname(os.path.realpath(__file__)) +FILE_PATH = os.path.join(DIR_PATH, "main.py") # `recorder -h` will show subcommands in the order they are added class NaturalOrderGroup(click.Group): @@ -21,3 +24,7 @@ def cli(): cli.add_command(reconstruct) cli.add_command(compute_tf) cli.add_command(apply_inv_tf) + + +if __name__ == '__main__': + cli() \ No newline at end of file diff --git a/recOrder/cli/parsing.py b/recOrder/cli/parsing.py index d302bd80..d32f2ace 100644 --- a/recOrder/cli/parsing.py +++ b/recOrder/cli/parsing.py @@ -118,4 +118,17 @@ def decorator(f: Callable) -> Callable: help="SLURM RAM multiplier.", )(f) + return decorator + +def unique_id() -> Callable: + def decorator(f: Callable) -> Callable: + return click.option( + "--unique-id", + "-uid", + default="", + required=False, + type=str, + help="Unique ID.", + )(f) + return decorator \ No newline at end of file diff --git a/recOrder/cli/reconstruct.py b/recOrder/cli/reconstruct.py index 4dcd8aae..9c77c11e 100644 --- a/recOrder/cli/reconstruct.py +++ b/recOrder/cli/reconstruct.py @@ -14,6 +14,7 @@ output_dirpath, processes_option, ram_multiplier, + unique_id, ) @@ -23,12 +24,14 @@ @output_dirpath() @processes_option(default=1) @ram_multiplier() +@unique_id() def reconstruct( input_position_dirpaths, config_filepath, output_dirpath, num_processes, ram_multiplier, + unique_id, ): """ Reconstruct a dataset using a configuration file. This is a @@ -65,4 +68,5 @@ def reconstruct( output_dirpath, num_processes, ram_multiplier, + unique_id, ) diff --git a/recOrder/io/utils.py b/recOrder/io/utils.py index c29769cc..229ac496 100644 --- a/recOrder/io/utils.py +++ b/recOrder/io/utils.py @@ -154,8 +154,8 @@ def yaml_to_model(yaml_path: Path, model): Examples -------- - >>> from my_model import MyModel - >>> model = yaml_to_model('model.yaml', MyModel) + # >>> from my_model import MyModel + # >>> model = yaml_to_model('model.yaml', MyModel) """ yaml_path = Path(yaml_path) diff --git a/recOrder/plugin/gui.py b/recOrder/plugin/gui.py index 04fd0f35..9f03ccb3 100644 --- a/recOrder/plugin/gui.py +++ b/recOrder/plugin/gui.py @@ -925,7 +925,7 @@ def setupUi(self, Form): self.gridLayout_6.addWidget(self.scrollArea_4, 4, 0, 1, 1) self.tabWidget.addTab(self.Acquisition, "") - self.recon_tab = tab_recon.Ui_Form() + self.recon_tab = tab_recon.Ui_Form(Form) self.tabWidget.addTab(self.recon_tab.recon_tab_widget, 'Reconstruction') self.Display = QtWidgets.QWidget() diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index dd02c054..4077186f 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1,11 +1,15 @@ -import os, json +import os, json, subprocess, time from pathlib import Path +import pydantic.v1 from qtpy import QtCore - -from qtpy.QtCore import Qt +from qtpy.QtCore import Qt, QEvent, QThread from qtpy.QtWidgets import * from magicgui.widgets import * +from PyQt5.QtCore import pyqtSignal + +from iohub.ngff import Plate, open_ome_zarr +from natsort import natsorted import pydantic, datetime, uuid from typing import Union, Literal @@ -15,20 +19,28 @@ import warnings from recOrder.io import utils -from recOrder.cli import settings +from recOrder.cli import settings, main, jobs_mgmt from napari.utils import notifications -from click.testing import CliRunner -from recOrder.cli.main import cli +from recOrder.cli.apply_inverse_transfer_function import ( + apply_inverse_transfer_function_cli, +) from concurrent.futures import ThreadPoolExecutor -STATUS_submitted = "Submitted" -STATUS_running = "Running" -STATUS_finished = "Finished" -STATUS_errored = "Errored" - +STATUS_submitted_pool = "Submitted_Pool" +STATUS_submitted_job = "Submitted_Job" +STATUS_running_pool = "Running_Pool" +STATUS_running_job = "Running_Job" +STATUS_finished_pool = "Finished_Pool" +STATUS_finished_job = "Finished_Job" +STATUS_errored_pool = "Errored_Pool" +STATUS_errored_job = "Errored_Job" + MSG_SUCCESS = {'msg':'success'} +JOB_COMPLETION_STR = "Job completed successfully" +JOB_RUNNING_STR = "Starting with JobEnvironment" +JOB_TRIGGERED_EXC = "Submitted job triggered an exception" # For now replicate CLI processing modes - these could reside in the CLI settings file as well # for consistency @@ -42,15 +54,17 @@ # Not efficient since instantiated from GUI # Does not have access to common functions in main_widget # ToDo : From main_widget and pass self reference -class Ui_Form(object): +class Ui_Form(QWidget): - def __init__(self): - super().__init__() + def __init__(self, parent=None): + super().__init__(parent) self.current_dir_path = str(Path.cwd()) self.current_save_path = str(Path.cwd()) self.input_directory = str(Path.cwd()) self.save_directory = str(Path.cwd()) + self.model_directory = str(Path.cwd()) self.yaml_model_file = str(Path.cwd()) + self._ui = parent # Top level parent self.recon_tab_widget = QWidget() @@ -58,7 +72,30 @@ def __init__(self): self.recon_tab_layout.setAlignment(Qt.AlignTop) self.recon_tab_layout.setContentsMargins(0,0,0,0) self.recon_tab_layout.setSpacing(0) - self.recon_tab_widget.setLayout(self.recon_tab_layout) + self.recon_tab_widget.setLayout(self.recon_tab_layout) + + # Top level - Data Input + self.modes_widget2 = QWidget() + self.modes_layout2 = QHBoxLayout() + self.modes_layout2.setAlignment(Qt.AlignTop) + self.modes_widget2.setLayout(self.modes_layout2) + self.modes_widget2.setMaximumHeight(50) + self.modes_widget2.setMinimumHeight(50) + + self.reconstruction_input_data_loc = widgets.LineEdit( + name="", + value=self.input_directory + ) + self.reconstruction_input_data_btn = widgets.PushButton( + name="InputData", + label="Input Data" + ) + self.reconstruction_input_data_btn.clicked.connect(self.browse_dir_path_input) + self.reconstruction_input_data_loc.changed.connect(self.readAndSetInputPathOnValidation) + + self.modes_layout2.addWidget(self.reconstruction_input_data_loc.native) + self.modes_layout2.addWidget(self.reconstruction_input_data_btn.native) + self.recon_tab_layout.addWidget(self.modes_widget2) # Top level - Selection modes, model creation and running self.modes_widget = QWidget() @@ -91,8 +128,8 @@ def __init__(self): # PushButton to validate and create the yaml file(s) based on selection self.build_button = widgets.PushButton(name="Build && Run Model") - self.build_button.clicked.connect(self.display_json_callback) - + self.build_button.clicked.connect(self.build_model_and_run) + # PushButton to clear all copies of models that are create for UI self.reconstruction_mode_clear = widgets.PushButton( name="ClearModels", @@ -109,29 +146,6 @@ def __init__(self): self.modes_layout.addWidget(self.reconstruction_mode_clear.native) self.recon_tab_layout.addWidget(self.modes_widget) - # Top level - Data Input - self.modes_widget2 = QWidget() - self.modes_layout2 = QHBoxLayout() - self.modes_layout2.setAlignment(Qt.AlignTop) - self.modes_widget2.setLayout(self.modes_layout2) - self.modes_widget2.setMaximumHeight(50) - self.modes_widget2.setMinimumHeight(50) - - self.reconstruction_input_data_loc = widgets.LineEdit( - name="", - value=self.input_directory - ) - self.reconstruction_input_data_btn = widgets.PushButton( - name="InputData", - label="Input Data" - ) - self.reconstruction_input_data_btn.clicked.connect(self.browse_dir_path_input) - self.reconstruction_input_data_loc.changed.connect(self.readAndSetInputPathOnValidation) - - self.modes_layout2.addWidget(self.reconstruction_input_data_loc.native) - self.modes_layout2.addWidget(self.reconstruction_input_data_btn.native) - self.recon_tab_layout.addWidget(self.modes_widget2) - _load_model_loc = widgets.LineEdit( name="", value=self.input_directory @@ -154,6 +168,16 @@ def __init__(self): _hBox_layout_model.addWidget(_load_model_loc.native) _hBox_layout_model.addWidget(_load_model_btn.native) self.recon_tab_layout.addWidget(_hBox_widget_model) + + # Line seperator between pydantic UI components + _line = QFrame() + _line.setMinimumWidth(1) + _line.setFixedHeight(2) + _line.setFrameShape(QFrame.HLine) + _line.setFrameShadow(QFrame.Sunken) + _line.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) + _line.setStyleSheet("margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;") + self.recon_tab_layout.addWidget(_line) # Top level - Central scrollable component which will hold Editable/(vertical) Expanding UI self.recon_tab_scrollArea_settings = QScrollArea() @@ -166,10 +190,7 @@ def __init__(self): self.recon_tab_qwidget_settings.setLayout(self.recon_tab_qwidget_settings_layout) self.recon_tab_scrollArea_settings.setWidget(self.recon_tab_qwidget_settings) self.recon_tab_layout.addWidget(self.recon_tab_scrollArea_settings) - - # Temp placeholder component to display, json pydantic output, validation msg, etc - # ToDo: Move to plugin message/error handling - + _scrollArea = QScrollArea() # _scrollArea.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) _scrollArea.setWidgetResizable(True) @@ -190,21 +211,72 @@ def __init__(self): _proc_table_widget.setLayout(self.proc_table_QFormLayout) _qwidget_settings_layout.addWidget(_proc_table_widget) - self.worker = MyWorker() - + # Stores Model values which cause validation failure - can be highlighted on the model field as Red + self.modelHighlighterVals = {} + + # Flag to delete Process update table row on successful Job completion + # self.autoDeleteRowOnCompletion = True + + self.worker = MyWorker(self.proc_table_QFormLayout) + + app = QApplication.instance() + app.lastWindowClosed.connect(self.myCloseEvent) # this line is connection to signal + + # our defined close event since napari doesnt do + def myCloseEvent(self): + event = QEvent(QEvent.Type.Close) + self.closeEvent(event) + + # on napari close - cleanup + def closeEvent(self, event): + if event.type() == QEvent.Type.Close: + self.worker.stopServer() + + def hideEvent(self, event): + if event.type() == QEvent.Type.Hide: + pass + + def showEvent(self, event): + if event.type() == QEvent.Type.Show: + pass + # Copied from main_widget # ToDo: utilize common functions # Input data selector def browse_dir_path_input(self): - result = self._open_file_dialog(self.current_dir_path, "dir") + result = self._open_file_dialog(self.input_directory, "dir") if result == '': return + + ret, ret_msg = self.validateInputData(result) + if not ret: + self.messageBox(ret_msg) + return + self.directory = result self.current_dir_path = result self.input_directory = result self.reconstruction_input_data_loc.value = self.input_directory + # not working - not used + def validateInputData(self, input_data_folder: str) -> list[Path]: + # Sort and validate the input paths, expanding plates into lists of positions + return True, MSG_SUCCESS + try: + input_paths = [Path(path) for path in natsorted(input_data_folder)] + for path in input_paths: + with open_ome_zarr(path, mode="r") as dataset: + if isinstance(dataset, Plate): + plate_path = input_paths.pop() + for position in dataset.positions(): + input_paths.append(plate_path / position[0]) + + return True, MSG_SUCCESS + except Exception as exc: + return False, exc.args + # call back for input LineEdit path changed manually + # include data validation def readAndSetInputPathOnValidation(self): if self.reconstruction_input_data_loc.value is None or len(self.reconstruction_input_data_loc.value) == 0: self.reconstruction_input_data_loc.value = self.input_directory @@ -221,7 +293,7 @@ def readAndSetInputPathOnValidation(self): # ToDo: utilize common functions # Output data selector def browse_dir_path_output(self, elem): - result = self._open_file_dialog(self.current_dir_path, "save") + result = self._open_file_dialog(self.save_directory, "save") if result == '': return self.directory = result @@ -241,53 +313,126 @@ def readAndSetOutputPathOnValidation(self, elem): # ToDo: utilize common functions # Output data selector def browse_dir_path_model(self, elem): - result = self._open_file_dialog(self.current_dir_path, "file") - if result == '': + results = self._open_file_dialog(self.model_directory, "files") # returns list + if len(results) == 0 or results == '': return - self.directory = result - self.current_dir_path = result - self.yaml_model_file = result - elem.value = self.yaml_model_file + + self.model_directory = str(Path(results[0]).parent.absolute()) + self.directory = self.model_directory + self.current_dir_path = self.model_directory + + pydantic_models = list() + for result in results: + self.yaml_model_file = result + elem.value = self.yaml_model_file + + with open(result, 'r') as yaml_in: + yaml_object = utils.yaml.safe_load(yaml_in) # yaml_object will be a list or a dict + jsonString = json.dumps(self.convert(yaml_object)) + json_out = json.loads(jsonString) + json_dict = dict(json_out) + + selected_modes = list(OPTION_TO_MODEL_DICT.copy().keys()) + exclude_modes = list(OPTION_TO_MODEL_DICT.copy().keys()) + + for k in range(len(selected_modes)-1, -1, -1): + if selected_modes[k] in json_dict.keys(): + exclude_modes.pop(k) + else: + selected_modes.pop(k) + + pruned_pydantic_class, ret_msg = self.buildModel(selected_modes) + if pruned_pydantic_class is None: + self.messageBox(ret_msg) + return - with open(result, 'r') as yaml_in: - yaml_object = utils.yaml.safe_load(yaml_in) # yaml_object will be a list or a dict - jsonString = json.dumps(self.convert(yaml_object)) - json_out = json.loads(jsonString) - json_dict = dict(json_out) + pydantic_model, ret_msg = self.get_model_from_file(self.yaml_model_file) + if pydantic_model is None: + self.messageBox(ret_msg) + return + + pydantic_model = self._create_acq_contols2(selected_modes, exclude_modes, pydantic_model, json_dict) + if pydantic_model is None: + self.messageBox("Error - pydantic model returned None") + return + + pydantic_models.append(pydantic_model) + + return pydantic_models + + # marks fields on the Model that cause a validation error + def modelHighlighter(self, errs, container:Container): + try: + self.modelHighlighterVals = {} + containerID = container.name + self.modelHighlighterVals[containerID] = {} + + errsList = list() + for err in errs: + errsList.append({"loc": err["loc"], "tooltip": err["msg"]}) - selected_modes = list(OPTION_TO_MODEL_DICT.copy().keys()) - exclude_modes = list(OPTION_TO_MODEL_DICT.copy().keys()) + self.modelHighlighterVals[containerID]["errs"] = errsList + self.modelHighlighterVals[containerID]["items"] = [] + + self.modelHighlighterSetter(errsList, container, containerID) + except Exception as exc: + print(exc.args) + # more of a test feature - no need to show up - for k in range(len(selected_modes)-1, -1, -1): - if selected_modes[k] in json_dict.keys(): - exclude_modes.pop(k) - else: - selected_modes.pop(k) + # recursively fix the container for highlighting + def modelHighlighterSetter(self, errs, container:Container, containerID): + try: + layout = container.native.layout() + for i in range(layout.count()): + item = layout.itemAt(i) + if item.widget(): + widget = layout.itemAt(i).widget() + if not isinstance(widget._magic_widget, CheckBox) and isinstance(widget._magic_widget._inner_widget, Container) and not (widget._magic_widget._inner_widget is None): + self.modelHighlighterSetter(errs, widget._magic_widget._inner_widget, containerID) + else: + for err in errs: + if isinstance(widget._magic_widget, CheckBox): + if widget._magic_widget.label == err["loc"][len(err["loc"])-1].replace("_", " "): + widget._magic_widget.tooltip = err["tooltip"] + widget._magic_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") + self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget) + return + elif widget._magic_widget._label_widget.value == err["loc"][len(err["loc"])-1].replace("_", " "): + widget._magic_widget._label_widget.tooltip = err["tooltip"] + widget._magic_widget._label_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") + self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget._label_widget) + widget._magic_widget._inner_widget.tooltip = err["tooltip"] + widget._magic_widget._inner_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") + self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget._inner_widget) + return + except Exception as exc: + print(exc.args) + # more of a test feature - no need to show up - pruned_pydantic_class, ret_msg = self.buildModel(selected_modes) - if pruned_pydantic_class is None: - self.messageBox(ret_msg) - return + # recursively fix the container for highlighting + def modelResetHighlighterSetter(self): + try: + for containerID in self.modelHighlighterVals.keys(): + items = self.modelHighlighterVals[containerID]["items"] + for widItem in items: + # widItem.tooltip = None # let them tool tip remain + widItem.native.setStyleSheet("border:1px solid rgb(0, 0, 0); border-width: 0px;") + + except Exception as exc: + print(exc.args) + # more of a test feature - no need to show up - pydantic_model, ret_msg = self.get_model_from_file(self.yaml_model_file) - if pydantic_model is None: - self.messageBox(ret_msg) - return - - pydantic_model = self._create_acq_contols2(selected_modes, exclude_modes, pydantic_model, json_dict) - if pydantic_model is None: - self.messageBox("Error - pydantic model returned None") - return - - return pydantic_model - # passes msg to napari notifications def messageBox(self, msg, type="exc"): if len(msg) > 0: try: json_object = msg - json_txt = json_object["loc"] + " >> " + json_object["msg"] + json_txt = "" + for err in json_object: + json_txt = json_txt + "Loc: {loc}\nMsg:{msg}\nType:{type}\n\n".format(loc=err["loc"], msg=err["msg"], type=err["type"]) + json_txt = str(json_txt) # ToDo: format it better + # formatted txt does not show up properly in msg-box ?? except: json_txt = str(msg) @@ -303,7 +448,8 @@ def messageBox(self, msg, type="exc"): # Running, Finished, Errored def addTableEntry(self, tableEntryID, tableEntryShortDesc, tableEntryVals, proc_params): - _scrollAreaCollapsibleBoxDisplayWidget = widgets.Label(value=tableEntryVals) # ToDo: Replace with tablular data and Stop button + _txtForInfoBox = "Updating {id}: Please wait...".format(id=tableEntryID) + _scrollAreaCollapsibleBoxDisplayWidget = widgets.Label(value=_txtForInfoBox) # ToDo: Replace with tablular data and Stop button _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBoxDisplayWidget.native) @@ -334,11 +480,16 @@ def addTableEntry(self, tableEntryID, tableEntryShortDesc, tableEntryVals, proc_ _expandingTabEntryWidget.layout().setContentsMargins(0,0,0,0) _expandingTabEntryWidget.layout().setSpacing(0) _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + + _scrollAreaCollapsibleBoxDisplayWidget.changed.connect(lambda: self.removeTableRow(_expandingTabEntryWidget, _scrollAreaCollapsibleBoxDisplayWidget)) - self.proc_table_QFormLayout.addRow(_expandingTabEntryWidget) + # instead of adding, insert at 0 to keep latest entry on top + # self.proc_table_QFormLayout.addRow(_expandingTabEntryWidget) + self.proc_table_QFormLayout.insertRow(0, _expandingTabEntryWidget) proc_params["table_layout"] = self.proc_table_QFormLayout proc_params["table_entry"] = _expandingTabEntryWidget + proc_params["table_entry_infoBox"] = _scrollAreaCollapsibleBoxDisplayWidget self.worker.runInPool(proc_params) # result = self.worker.getResult(proc_params["exp_id"]) @@ -369,12 +520,18 @@ def buildModel(self, selected_modes): f = settings.FluorescenceSettings() exclude_modes = ["birefringence", "phase"] - model = settings.ReconstructionSettings(input_channel_names=chNames, birefringence=b, phase=p, fluorescence=f) - model = self._fix_model(model, exclude_modes, 'input_channel_names', chNames) + model = None + try: + model = settings.ReconstructionSettings(input_channel_names=chNames, birefringence=b, phase=p, fluorescence=f) + except pydantic.v1.ValidationError as exc: + # use v1 for ValidationError - newer one is not caught properly + return None, exc.errors() + model = self._fix_model(model, exclude_modes, 'input_channel_names', chNames) return model, "+".join(selected_modes) + ": MSG_SUCCESS" - except pydantic.ValidationError as exc: - return None, "+".join(selected_modes) + str(exc.errors()[0]) + + except Exception as exc: + return None, exc.args # ToDo: Temporary fix to over ride the 'input_channel_names' default value # Needs revisitation @@ -417,9 +574,13 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None self.messageBox(ret_msg) return + # Final constant UI val and identifier + _idx: Final[int] = self.index + _str: Final[str] = str(uuid.uuid4()) + # Container holding the pydantic UI components # Multiple instances/copies since more than 1 might be created - recon_pydantic_container = widgets.Container(name="", scrollable=False) + recon_pydantic_container = widgets.Container(name=_str, scrollable=False) self.add_pydantic_to_container(pydantic_class, recon_pydantic_container, exclude_modes, json_dict) @@ -474,11 +635,7 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None # Use case when a wrong selection of input modes get selected eg Bire+Fl # Preferably this root level validation should occur before values arevalidated # in order to display and avoid this to occur - _del_button = widgets.PushButton(name="Delete this Model") - - # Final constant UI val and identifier - _idx: Final[int] = self.index - _str: Final[str] = uuid.uuid4() + _del_button = widgets.PushButton(name="Delete this Model") # Output Data location # These could be multiple based on user selection for each model @@ -520,6 +677,12 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None # uuid - used for identiying in editable list self.pydantic_classes.append({'uuid':_str, 'class':pydantic_class, 'input':self.reconstruction_input_data_loc, 'output':_output_data_loc, 'container':recon_pydantic_container, 'selected_modes':selected_modes.copy(), 'exclude_modes':exclude_modes.copy()}) self.index += 1 + + if self.index > 1: + self.build_button.text = "Build && Run {n} Models".format(n=self.index) + else: + self.build_button.text = "Build && Run Model" + return pydantic_model # UI components deletion - maybe just needs the parent container instead of individual components @@ -543,6 +706,11 @@ def _delete_model(self, wid1, wid2, wid3, wid4, wid5, index, _str): self.pydantic_classes.pop(i) return i += 1 + self.index = i + if self.index > 1: + self.build_button.text = "Build && Run {n} Models".format(n=self.index) + else: + self.build_button.text = "Build && Run Model" # Clear all the generated pydantic models and clears the pydantic model list def _clear_all_models(self): @@ -554,14 +722,18 @@ def _clear_all_models(self): index -=1 self.pydantic_classes.clear() self.index = 0 + self.build_button.text = "Build && Run Model" # Displays the json output from the pydantic model UI selections by user # Loops through all our stored pydantic classes - def display_json_callback(self): + def build_model_and_run(self): # we dont want to have a partial run if there are N models # so we will validate them all first and then run in a second loop # first pass for validating # second pass for creating yaml and processing + + self.modelResetHighlighterSetter() # reset the container elements that might be highlighted for errors + for item in self.pydantic_classes: cls = item['class'] cls_container = item['container'] @@ -594,8 +766,10 @@ def display_json_callback(self): pydantic_kwargs["time_indices"] = time_indices # validate and return errors if None + # ToDo: error causing fields could be marked red pydantic_model, ret_msg = self.validate_pydantic_model(cls, pydantic_kwargs) if pydantic_model is None: + self.modelHighlighter(ret_msg, cls_container) self.messageBox(ret_msg) return @@ -684,9 +858,11 @@ def display_json_callback(self): proc_params = {} proc_params["exp_id"] = expID + proc_params["desc"] = tableDescToolTip proc_params["config_path"] = str(Path(config_path).absolute()) proc_params["input_path"] = str(Path(input_dir).absolute()) proc_params["output_path"] = str(Path(output_dir).absolute()) + proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) self.addTableEntry(tableID, tableDescToolTip, json_txt, proc_params) @@ -753,19 +929,25 @@ def clean_string_int_for_list(self, field, string): # get the pydantic_kwargs and catches any errors in doing so def get_and_validate_pydantic_args(self, cls_container, cls, pydantic_kwargs, exclude_modes): try: - self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs, exclude_modes) - return pydantic_kwargs, MSG_SUCCESS - except pydantic.ValidationError as exc: - return None, exc.errors()[0] + try: + self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs, exclude_modes) + return pydantic_kwargs, MSG_SUCCESS + except pydantic.v1.ValidationError as exc: + return None, exc.errors() + except Exception as exc: + return None, exc.args # validate the model and return errors for user actioning def validate_pydantic_model(self, cls, pydantic_kwargs): # instantiate the pydantic model form the kwargs we just pulled - try : - pydantic_model = settings.ReconstructionSettings.parse_obj(pydantic_kwargs) - return pydantic_model, MSG_SUCCESS - except pydantic.ValidationError as exc: - return None, exc.errors()[0] + try: + try : + pydantic_model = settings.ReconstructionSettings.parse_obj(pydantic_kwargs) + return pydantic_model, MSG_SUCCESS + except pydantic.v1.ValidationError as exc: + return None, exc.errors() + except Exception as exc: + return None, exc.args # test to make sure model coverts to json which should ensure compatibility with yaml export def validate_and_return_json(self, pydantic_model): @@ -816,7 +998,7 @@ def add_pydantic_to_container(self, py_model:Union[pydantic.BaseModel, pydantic. if field_def is not None and field not in excludes: def_val = field_def.default ftype = field_def.type_ - if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass): + if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass) or isinstance(ftype, pydantic.v1.main.ModelMetaclass): json_val = None if json_dict is not None: json_val = json_dict[field] @@ -865,7 +1047,7 @@ def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pyda for field, field_def in pydantic_model.__fields__.items(): if field_def is not None and field not in excludes: ftype = field_def.type_ - if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass): + if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass) or isinstance(ftype, pydantic.v1.main.ModelMetaclass): # go deeper pydantic_kwargs[field] = {} # new dictionary for the new nest level # any pydantic class will be a container, so pull that out to pass @@ -885,6 +1067,8 @@ def _open_file_dialog(self, default_path, type): return self._open_dialog("select a directory", str(default_path), type) elif type == "file": return self._open_dialog("select a file", str(default_path), type) + elif type == "files": + return self._open_dialog("select file(s)", str(default_path), type) elif type == "save": return self._open_dialog("save a file", str(default_path), type) else: @@ -914,6 +1098,10 @@ def _open_dialog(self, title, ref, type): path = QFileDialog.getOpenFileName( None, title, ref, options=options )[0] + elif type == "files": + path = QFileDialog.getOpenFileNames( + None, title, ref, options=options + )[0] elif type == "save": path = QFileDialog.getSaveFileName( None, "Choose a save name", ref, options=options @@ -995,18 +1183,98 @@ def setContentLayout(self, layout): content_animation.setStartValue(0) content_animation.setEndValue(content_height) -class MyWorker(object): - - def __init__(self): - super().__init__() +import socket, threading +class MyWorker(): + + def __init__(self, formLayout): + super().__init__() + self.formLayout = formLayout self.max_cores = os.cpu_count() # In the case of CLI, we just need to submit requests in a non-blocking way self.threadPool = int(self.max_cores/2) self.results = {} - self.pool = None + self.pool = None # https://click.palletsprojects.com/en/stable/testing/ - self.runner = CliRunner() + # self.runner = CliRunner() self.startPool() + # jobs_mgmt.shared_var_jobs = self.JobsManager.shared_var_jobs + self.lock = threading.Lock() + self.JobsMgmt = jobs_mgmt.JobsManagement() + self.useServer = True + self.serverRunning = True + self.serverSocket = None + thread = threading.Thread(target=self.startServer) + thread.start() + + def findWidgetRowInLayout(self, strID): + layout: QFormLayout = self.formLayout + for idx in range(0, layout.rowCount()): + widgetItem = layout.itemAt(idx) + name_widget = widgetItem.widget() + toolTip_string = str(name_widget.toolTip) + if strID in toolTip_string: + name_widget.setParent(None) + return idx + return -1 + + def removeRow(self, row, expID): + try: + with self.lock: + if row < self.formLayout.rowCount(): + layout: QFormLayout = self.formLayout + widgetItem = layout.itemAt(row) + name_widget = widgetItem.widget() + toolTip_string = str(name_widget.toolTip) + if expID in toolTip_string: + layout.removeRow(row) + except Exception as exc: + print(exc.args) + + def startServer(self): + try: + if not self.useServer: + return + + self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.serverSocket.bind(('localhost', jobs_mgmt.SERVER_PORT)) + self.serverSocket.listen(50) # become a server socket, maximum 50 connections + + while self.serverRunning: + connection, address = self.serverSocket.accept() + try: + buf = connection.recv(64) + if len(buf) > 0: + decoded_string = buf.decode("utf-8") + json_str = str(decoded_string) + json_obj = json.loads(json_str) + uID = "" + jobID = "" + for k in json_obj: + self.JobsMgmt.shared_var_jobs[k] = json_obj[k] + uID = k + jobID = json_obj[k] + + # dont block the server thread + thread = threading.Thread(target=self.tableUpdateAndCleaupThread, args=(uID, jobID)) + thread.start() + except Exception as exc: + print(exc.args) + time.sleep(1) + + self.serverSocket.close() + except Exception as exc: + if not self.serverRunning: + self.serverRunning = True + return # ignore - will cause an exception on napari close but that is fine and does the job + print(exc.args) + + def stopServer(self): + try: + if self.serverSocket is not None: + self.serverRunning = False + self.serverSocket.close() + except Exception as exc: + print(exc.args) def getMaxCPU_cores(self): return self.max_cores @@ -1022,18 +1290,104 @@ def startPool(self): def shutDownPool(self): self.pool.shutdown(wait=False) + # the table update thread can be called from multiple points/threads + # on errors - table row item is updated but there is no row deletion + # on successful processing - the row item is expected to be deleted + # row is being deleted from a seperate thread for which we need to connect using signal + def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): + # finished will be updated by the job - submitit status + + if expIdx != "" and jobIdx != "": + # this request came from server so we can wait for the Job to finish and update progress + params = self.results[expIdx] + _infoBox = params["table_entry_infoBox"] + _txtForInfoBox = "Updating {id}: Please wait... \nJobID assigned: {jID} ".format(id=params["desc"], jID=jobIdx) + _infoBox.value = _txtForInfoBox + while True: + time.sleep(1) # update every sec and exit on break + if self.JobsMgmt.hasSubmittedJob(expIdx): + if params["status"] in [STATUS_finished_job]: + break + elif params["status"] in [STATUS_errored_job]: + jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") + _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobERR + break + else: + jobTXT = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="out") + try: + if jobTXT == "": # job file not created yet + time.sleep(2) + elif self.results[expIdx]["status"] == STATUS_finished_job: + rowIdx = self.findWidgetRowInLayout(expIdx) + # check to ensure row deletion due to shrinking table + # if not deleted try to delete again + if rowIdx < 0: + break + else: + self.workerThread = RowDeletionWorkerThread(self.formLayout, expIdx) + self.workerThread.removeRowSignal.connect(self.removeRow) + self.workerThread.start() + elif JOB_COMPLETION_STR in jobTXT: + self.results[expIdx]["status"] = STATUS_finished_job + _infoBox = params["table_entry_infoBox"] + _infoBox.value = jobTXT + # this is the only case where row deleting occurs + # we cant delete the row directly from this thread + # we will use the exp_id to identify and delete the row + # using pyqtSignal + self.workerThread = RowDeletionWorkerThread(self.formLayout, expIdx) + self.workerThread.removeRowSignal.connect(self.removeRow) + self.workerThread.start() + time.sleep(2) + # break - based on status + elif JOB_RUNNING_STR in jobTXT: + self.results[expIdx]["status"] = STATUS_running_job + _infoBox = params["table_entry_infoBox"] + _infoBox.value = jobTXT + elif JOB_TRIGGERED_EXC in jobTXT: + self.results[expIdx]["status"] = STATUS_errored_job + jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") + _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobTXT +"\n\n"+ jobERR + else: + jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") + _infoBox = params["table_entry_infoBox"] + _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobERR + break + except Exception as exc: + print(exc.args) + else: + # this would occur when an exception happens on the pool side before or during job submission + # we dont have a job ID and will update based on exp_ID/uIU + for param_ID in self.results.keys(): + params = self.results[param_ID] + if params["status"] in [STATUS_errored_pool]: + _infoBox = params["table_entry_infoBox"] + poolERR = self.results[params["exp_id"]]["error"] + _infoBox.value = poolERR + def runInPool(self, params): self.results[params["exp_id"]] = params - self.results[params["exp_id"]]["status"] = STATUS_submitted + self.results[params["exp_id"]]["status"] = STATUS_running_pool self.results[params["exp_id"]]["error"] = "" - self.pool.submit(self.run, params) + try: + self.pool.submit(self.run, params) + except Exception as exc: + self.results[params["exp_id"]]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["error"] = str(exc.args) + self.tableUpdateAndCleaupThread() def runMultiInPool(self, multi_params_as_list): for params in multi_params_as_list: self.results[params["exp_id"]] = params - self.results[params["exp_id"]]["status"] = STATUS_submitted + self.results[params["exp_id"]]["status"] = STATUS_submitted_pool self.results[params["exp_id"]]["error"] = "" - self.pool.map(self.run, multi_params_as_list) + try: + self.pool.map(self.run, multi_params_as_list) + except Exception as exc: + for params in multi_params_as_list: + self.results[params["exp_id"]]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["error"] = str(exc.args) + self.tableUpdateAndCleaupThread() def getResults(self): return self.results @@ -1047,37 +1401,70 @@ def run(self, params): if params["exp_id"] not in self.results.keys(): self.results[params["exp_id"]] = params self.results[params["exp_id"]]["error"] = "" + + try: + # does need further threading ? probably not ! + thread = threading.Thread(target=self.runInSubProcess, args=(params,)) + thread.start() - self.results[params["exp_id"]]["status"] = STATUS_running - try: - input_path = params["input_path"] - config_path = params["config_path"] - output_path = params["output_path"] - - # ToDo: replace with command line ver - result = self.runner.invoke( - cli, - [ - "reconstruct", - "-i", - str(input_path), - "-c", - str(config_path), - "-o", - str(output_path), - ], - catch_exceptions=False, - ) + # self.runInSubProcess(params) + + # check for this job to show up in submitit jobs list + # wait for 2 sec before raising an error + + except Exception as exc: + self.results[params["exp_id"]]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["error"] = str(exc.args) + self.tableUpdateAndCleaupThread() + + def runInSubProcess(self, params): + try: + input_path = str(params["input_path"]) + config_path = str(params["config_path"]) + output_path = str(params["output_path"]) + uid = str(params["exp_id"]) + mainfp = str(main.FILE_PATH) + + self.results[params["exp_id"]]["status"] = STATUS_submitted_job - self.results[params["exp_id"]]["result"] = result - self.results[params["exp_id"]]["status"] = STATUS_finished + proc = subprocess.run(['python', mainfp, 'reconstruct', '-i', input_path, '-c', config_path, '-o', output_path, '-uid', uid]) + self.results[params["exp_id"]]["proc"] = proc + if proc.returncode != 0: + raise Exception("An error occurred in processing ! Check terminal output.") - _proc_table_QFormLayout = self.results[params["exp_id"]]["table_layout"] - _expandingTabEntryWidget = self.results[params["exp_id"]]["table_entry"] - _proc_table_QFormLayout.removeRow(_expandingTabEntryWidget) except Exception as exc: - self.results[params["exp_id"]]["status"] = STATUS_errored - self.results[params["exp_id"]]["error"] = exc.args + self.results[params["exp_id"]]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["error"] = str(exc.args) + self.tableUpdateAndCleaupThread() + +# Emits a signal to QFormLayout on the main thread +class RowDeletionWorkerThread(QThread): + removeRowSignal = pyqtSignal(int, str) + + def __init__(self, formLayout, strID): + super().__init__() + self.formLayout = formLayout + self.deleteRow = -1 + self.stringID = strID + + # we might deal with race conditions with a shrinking table + # find out widget and return its index + def findWidgetRowInLayout(self, strID): + layout: QFormLayout = self.formLayout + for idx in range(0, layout.rowCount()): + widgetItem = layout.itemAt(idx) + name_widget = widgetItem.widget() + toolTip_string = str(name_widget.toolTip) + if strID in toolTip_string: + name_widget.setParent(None) + return idx + return -1 + + def run(self): + # Emit the signal to remove the row + self.deleteRow = self.findWidgetRowInLayout(self.stringID) + if self.deleteRow > -1: + self.removeRowSignal.emit(int(self.deleteRow), str(self.stringID)) # VScode debugging if __name__ == "__main__": From e311496391e6400047b4fceb3bf88c6ce10bf13c Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Mon, 16 Dec 2024 19:35:47 -0500 Subject: [PATCH 04/38] possible fix for macOS - import OS specific pydantic imports for ModelMetaclass --- recOrder/plugin/tab_recon.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 4077186f..46741af4 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1,7 +1,6 @@ -import os, json, subprocess, time +import os, sys, json, subprocess, time, datetime, uuid from pathlib import Path -import pydantic.v1 from qtpy import QtCore from qtpy.QtCore import Qt, QEvent, QThread from qtpy.QtWidgets import * @@ -11,9 +10,7 @@ from iohub.ngff import Plate, open_ome_zarr from natsort import natsorted -import pydantic, datetime, uuid -from typing import Union, Literal -from typing import Final +from typing import Union, Literal, Final from magicgui import widgets from magicgui.type_map import get_widget_class import warnings @@ -22,12 +19,22 @@ from recOrder.cli import settings, main, jobs_mgmt from napari.utils import notifications -from recOrder.cli.apply_inverse_transfer_function import ( - apply_inverse_transfer_function_cli, -) - from concurrent.futures import ThreadPoolExecutor +import pydantic.v1, pydantic + +try: + if sys.platform == "win32": + # windows + from pydantic.v1.main import ModelMetaclass + elif sys.platform == "darwin": + # macOS + from pydantic.main import ModelMetaclass + elif sys.platform.startswith("linux"): + from pydantic.main import ModelMetaclass +except: + pass + STATUS_submitted_pool = "Submitted_Pool" STATUS_submitted_job = "Submitted_Job" STATUS_running_pool = "Running_Pool" @@ -990,7 +997,7 @@ def convert(self, obj): # excludes handles fields that are not supposed to show up from __fields__ # json_dict adds ability to provide new set of default values at time of container creation - def add_pydantic_to_container(self, py_model:Union[pydantic.BaseModel, pydantic.main.ModelMetaclass], container: widgets.Container, excludes=[], json_dict=None): + def add_pydantic_to_container(self, py_model:Union[pydantic.BaseModel, ModelMetaclass], container: widgets.Container, excludes=[], json_dict=None): # recursively traverse a pydantic model adding widgets to a container. When a nested # pydantic model is encountered, add a new nested container @@ -998,7 +1005,7 @@ def add_pydantic_to_container(self, py_model:Union[pydantic.BaseModel, pydantic. if field_def is not None and field not in excludes: def_val = field_def.default ftype = field_def.type_ - if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass) or isinstance(ftype, pydantic.v1.main.ModelMetaclass): + if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, ModelMetaclass) or isinstance(ftype, ModelMetaclass): json_val = None if json_dict is not None: json_val = json_dict[field] @@ -1047,7 +1054,7 @@ def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pyda for field, field_def in pydantic_model.__fields__.items(): if field_def is not None and field not in excludes: ftype = field_def.type_ - if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, pydantic.main.ModelMetaclass) or isinstance(ftype, pydantic.v1.main.ModelMetaclass): + if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, ModelMetaclass) or isinstance(ftype, ModelMetaclass): # go deeper pydantic_kwargs[field] = {} # new dictionary for the new nest level # any pydantic class will be a container, so pull that out to pass From ee8e09f83dc537d922bdeacddcb966bede7ba95b Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 17 Dec 2024 10:03:42 -0500 Subject: [PATCH 05/38] use ver specific fix for pydantic instead of OS - pydantic possibly needs to be pinned at 1.10.19 @talonchandler - implemented ver specific import for now --- recOrder/plugin/tab_recon.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 46741af4..ebb69a17 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -22,18 +22,23 @@ from concurrent.futures import ThreadPoolExecutor import pydantic.v1, pydantic +import importlib.metadata try: - if sys.platform == "win32": - # windows - from pydantic.v1.main import ModelMetaclass - elif sys.platform == "darwin": - # macOS - from pydantic.main import ModelMetaclass - elif sys.platform.startswith("linux"): + # Use version specific pydantic import for ModelMetaclass + # prefer to pin to 1.10.19 + version = importlib.metadata.version('pydantic') + # print("Your Pydantic library ver:{v}.".format(v=version)) + if version >= "2.0.0": + print("Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format(v=version)) + from pydantic.main import ModelMetaclass + elif version >= "1.10.19": from pydantic.main import ModelMetaclass + else: + print("Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format(v=version)) + from pydantic.v1.main import ModelMetaclass except: - pass + print("Pydantic library was not found. Ver 1.10.19 is recommended.") STATUS_submitted_pool = "Submitted_Pool" STATUS_submitted_job = "Submitted_Job" From ab03bd83a679bbe740714243daf3829b87080555 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 17 Dec 2024 11:21:39 -0500 Subject: [PATCH 06/38] set a timeout limit on job update thread - check for update on the job file, if not - notify user in process table and exit after a 2 min timeout --- recOrder/plugin/tab_recon.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index ebb69a17..5d2280b9 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1025,6 +1025,7 @@ def add_pydantic_to_container(self, py_model:Union[pydantic.BaseModel, ModelMeta elif isinstance(def_val, str): #field == "time_indices": new_widget_cls, ops = get_widget_class(None, str, dict(name=field, value=def_val)) new_widget = new_widget_cls(**ops) + new_widget.tooltip = field if isinstance(new_widget, widgets.EmptyWidget): warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") elif isinstance(def_val, float): @@ -1032,15 +1033,18 @@ def add_pydantic_to_container(self, py_model:Union[pydantic.BaseModel, ModelMeta if def_val > -1 and def_val < 1: new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val, step=float(0.001))) new_widget = new_widget_cls(**ops) + new_widget.tooltip = field_def.name else: new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val)) new_widget = new_widget_cls(**ops) + new_widget.tooltip = field_def.name if isinstance(new_widget, widgets.EmptyWidget): warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") else: # parse the field, add appropriate widget new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val)) new_widget = new_widget_cls(**ops) + new_widget.tooltip = field_def.name if isinstance(new_widget, widgets.EmptyWidget): warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") if json_dict is not None and not isinstance(new_widget, widgets.Container): @@ -1311,10 +1315,15 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): if expIdx != "" and jobIdx != "": # this request came from server so we can wait for the Job to finish and update progress + # some wait logic needs to be added otherwise for unknown errors this thread will persist + # perhaps set a time out limit and then update the status window and then exit params = self.results[expIdx] _infoBox = params["table_entry_infoBox"] _txtForInfoBox = "Updating {id}: Please wait... \nJobID assigned: {jID} ".format(id=params["desc"], jID=jobIdx) _infoBox.value = _txtForInfoBox + _tUpdateCount = 0 + _tUpdateCountTimeout = 120 # 2 mins + _lastUpdate_jobTXT = "" while True: time.sleep(1) # update every sec and exit on break if self.JobsMgmt.hasSubmittedJob(expIdx): @@ -1341,7 +1350,6 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): self.workerThread.start() elif JOB_COMPLETION_STR in jobTXT: self.results[expIdx]["status"] = STATUS_finished_job - _infoBox = params["table_entry_infoBox"] _infoBox.value = jobTXT # this is the only case where row deleting occurs # we cant delete the row directly from this thread @@ -1354,15 +1362,24 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): # break - based on status elif JOB_RUNNING_STR in jobTXT: self.results[expIdx]["status"] = STATUS_running_job - _infoBox = params["table_entry_infoBox"] _infoBox.value = jobTXT + _tUpdateCount += 1 + if _tUpdateCount > 60: + if _lastUpdate_jobTXT != jobTXT: + # if there is an update reset counter + _tUpdateCount=0 + _lastUpdate_jobTXT = jobTXT + else: + _infoBox.value = "Please check terminal output for Job status..\n\n" + jobTXT + if _tUpdateCount > _tUpdateCountTimeout: + break elif JOB_TRIGGERED_EXC in jobTXT: self.results[expIdx]["status"] = STATUS_errored_job jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobTXT +"\n\n"+ jobERR + break else: jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") - _infoBox = params["table_entry_infoBox"] _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobERR break except Exception as exc: From 4c7692d4aa79e4885da1cfe9bb9d355df290a9fc Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Wed, 18 Dec 2024 15:48:35 -0500 Subject: [PATCH 07/38] fixes & enhancements - fixes newly created yaml files to reside besides output zarrs - fixes Model numbers not correctly representing on Build & Run button - fixes "reconstruction algorithm" to show up as dropdown box - fixes "regularization strength" spinbox to accept 5 significant decimal place values - background path now accepts Path and str in model and shows up as directory selector button - each model container setting is now a collapsible item - fixes issues when widget is closed/hidden and then started again - all model(s) validation errors are not collected and presented in notification and highlighted on GUI in one go - each collapsible model entry also highlights if any validation errors are present - added confirm dialog for certain actions - added clear results button - job update thread will exit after a timeout limit and no update detected in job output file --- recOrder/cli/settings.py | 3 +- recOrder/plugin/tab_recon.py | 517 ++++++++++++++++++++++++++--------- 2 files changed, 391 insertions(+), 129 deletions(-) diff --git a/recOrder/cli/settings.py b/recOrder/cli/settings.py index a7661fd9..39056ceb 100644 --- a/recOrder/cli/settings.py +++ b/recOrder/cli/settings.py @@ -1,5 +1,6 @@ import os from typing import List, Literal, Optional, Union +from pathlib import Path from pydantic.v1 import ( BaseModel, @@ -40,7 +41,7 @@ def swing_range(cls, v): class BirefringenceApplyInverseSettings(WavelengthIllumination): - background_path: str = "" + background_path: Union[str, Path] = "" remove_estimated_background: bool = False flip_orientation: bool = False rotate_orientation: bool = False diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 5d2280b9..f75944e3 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1,4 +1,4 @@ -import os, sys, json, subprocess, time, datetime, uuid +import os, json, subprocess, time, datetime, uuid from pathlib import Path from qtpy import QtCore @@ -10,7 +10,7 @@ from iohub.ngff import Plate, open_ome_zarr from natsort import natsorted -from typing import Union, Literal, Final +from typing import List, Literal, Union, Final, Annotated from magicgui import widgets from magicgui.type_map import get_widget_class import warnings @@ -21,9 +21,19 @@ from concurrent.futures import ThreadPoolExecutor -import pydantic.v1, pydantic import importlib.metadata +import pydantic.v1, pydantic +from pydantic.v1 import ( + BaseModel, + Extra, + NonNegativeFloat, + NonNegativeInt, + PositiveFloat, + root_validator, + validator, +) + try: # Use version specific pydantic import for ModelMetaclass # prefer to pin to 1.10.19 @@ -32,11 +42,17 @@ if version >= "2.0.0": print("Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format(v=version)) from pydantic.main import ModelMetaclass + from pydantic.main import ValidationError + from pydantic.main import BaseModel elif version >= "1.10.19": from pydantic.main import ModelMetaclass + from pydantic.main import ValidationError + from pydantic.main import BaseModel else: print("Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format(v=version)) from pydantic.v1.main import ModelMetaclass + from pydantic.v1.main import ValidationError + from pydantic.v1.main import BaseModel except: print("Pydantic library was not found. Ver 1.10.19 is recommended.") @@ -54,6 +70,9 @@ JOB_RUNNING_STR = "Starting with JobEnvironment" JOB_TRIGGERED_EXC = "Submitted job triggered an exception" +_validate_alert = '⚠' +_validate_ok = '✔️' + # For now replicate CLI processing modes - these could reside in the CLI settings file as well # for consistency OPTION_TO_MODEL_DICT = { @@ -62,6 +81,12 @@ "fluorescence": {"enabled":False, "setting":None}, } +CONTAINERS_INFO = {} + +# This keeps an instance of the MyWorker server that is listening +# napari will not stop processes and the Hide event is not reliable +HAS_INSTANCE = {"val": False, "instance": None} + # Main class for the Reconstruction tab # Not efficient since instantiated from GUI # Does not have access to common functions in main_widget @@ -70,14 +95,23 @@ class Ui_Form(QWidget): def __init__(self, parent=None): super().__init__(parent) - self.current_dir_path = str(Path.cwd()) - self.current_save_path = str(Path.cwd()) - self.input_directory = str(Path.cwd()) - self.save_directory = str(Path.cwd()) - self.model_directory = str(Path.cwd()) - self.yaml_model_file = str(Path.cwd()) self._ui = parent + if HAS_INSTANCE["val"]: + self.current_dir_path = str(Path.cwd()) + self.current_save_path = HAS_INSTANCE["current_save_path"] + self.input_directory = HAS_INSTANCE["input_directory"] + self.save_directory = HAS_INSTANCE["save_directory"] + self.model_directory = HAS_INSTANCE["model_directory"] + self.yaml_model_file = HAS_INSTANCE["yaml_model_file"] + else: + self.current_dir_path = str(Path.cwd()) + self.current_save_path = str(Path.cwd()) + self.input_directory = str(Path.cwd()) + self.save_directory = str(Path.cwd()) + self.model_directory = str(Path.cwd()) + self.yaml_model_file = str(Path.cwd()) + # Top level parent self.recon_tab_widget = QWidget() self.recon_tab_layout = QVBoxLayout() @@ -160,15 +194,21 @@ def __init__(self, parent=None): _load_model_loc = widgets.LineEdit( name="", - value=self.input_directory + value=self.model_directory ) _load_model_btn = widgets.PushButton( name="LoadModel", label="Load Model" - ) + ) # Passing model location label to model location selector _load_model_btn.clicked.connect(lambda: self.browse_dir_path_model(_load_model_loc)) + + _clear_results_btn = widgets.PushButton( + name="ClearResults", + label="Clear Results" + ) + _clear_results_btn.clicked.connect(self.clear_results_table) # HBox for Loading Model _hBox_widget_model = QWidget() @@ -179,6 +219,7 @@ def __init__(self, parent=None): _hBox_widget_model.setMinimumHeight(50) _hBox_layout_model.addWidget(_load_model_loc.native) _hBox_layout_model.addWidget(_load_model_btn.native) + _hBox_layout_model.addWidget(_clear_results_btn.native) self.recon_tab_layout.addWidget(_hBox_widget_model) # Line seperator between pydantic UI components @@ -193,7 +234,7 @@ def __init__(self, parent=None): # Top level - Central scrollable component which will hold Editable/(vertical) Expanding UI self.recon_tab_scrollArea_settings = QScrollArea() - self.recon_tab_scrollArea_settings.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) + # self.recon_tab_scrollArea_settings.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.recon_tab_scrollArea_settings.setWidgetResizable(True) self.recon_tab_qwidget_settings = QWidget() self.recon_tab_qwidget_settings_layout = QVBoxLayout() @@ -202,7 +243,16 @@ def __init__(self, parent=None): self.recon_tab_qwidget_settings.setLayout(self.recon_tab_qwidget_settings_layout) self.recon_tab_scrollArea_settings.setWidget(self.recon_tab_qwidget_settings) self.recon_tab_layout.addWidget(self.recon_tab_scrollArea_settings) - + + _line2 = QFrame() + _line2.setMinimumWidth(1) + _line2.setFixedHeight(2) + _line2.setFrameShape(QFrame.HLine) + _line2.setFrameShadow(QFrame.Sunken) + _line2.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) + _line2.setStyleSheet("margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;") + self.recon_tab_layout.addWidget(_line2) + _scrollArea = QScrollArea() # _scrollArea.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) _scrollArea.setWidgetResizable(True) @@ -223,16 +273,23 @@ def __init__(self, parent=None): _proc_table_widget.setLayout(self.proc_table_QFormLayout) _qwidget_settings_layout.addWidget(_proc_table_widget) - # Stores Model values which cause validation failure - can be highlighted on the model field as Red + # Stores Model & Components values which cause validation failure - can be highlighted on the model field as Red self.modelHighlighterVals = {} # Flag to delete Process update table row on successful Job completion # self.autoDeleteRowOnCompletion = True - self.worker = MyWorker(self.proc_table_QFormLayout) + # handle napari's close widget and avoid starting a second server + if HAS_INSTANCE["val"]: + self.worker:MyWorker = HAS_INSTANCE["MyWorker"] + self.worker.setNewInstances(self.proc_table_QFormLayout, self._ui) + else: + self.worker = MyWorker(self.proc_table_QFormLayout, self._ui) + HAS_INSTANCE["val"] = True + HAS_INSTANCE["MyWorker"] = self.worker app = QApplication.instance() - app.lastWindowClosed.connect(self.myCloseEvent) # this line is connection to signal + app.lastWindowClosed.connect(self.myCloseEvent) # this line is connection to signal close # our defined close event since napari doesnt do def myCloseEvent(self): @@ -245,13 +302,21 @@ def closeEvent(self, event): self.worker.stopServer() def hideEvent(self, event): - if event.type() == QEvent.Type.Hide: + if event.type() == QEvent.Type.Hide and self._ui.isVisible(): pass def showEvent(self, event): if event.type() == QEvent.Type.Show: pass + def confirmDialog(self): + qm = QMessageBox + ret = qm.question(self.recon_tab_widget, "Confirm", "Confirm your selection ?", qm.Yes | qm.No) + if ret == qm.Yes: + return True + else: + return False + # Copied from main_widget # ToDo: utilize common functions # Input data selector @@ -268,7 +333,21 @@ def browse_dir_path_input(self): self.directory = result self.current_dir_path = result self.input_directory = result - self.reconstruction_input_data_loc.value = self.input_directory + self.reconstruction_input_data_loc.value = result + + self.saveLastPaths() + + def browse_dir_path_inputBG(self, elem): + result = self._open_file_dialog(self.directory, "dir") + if result == '': + return + + ret, ret_msg = self.validateInputData(result) + if not ret: + self.messageBox(ret_msg) + return + + elem.value = result # not working - not used def validateInputData(self, input_data_folder: str) -> list[Path]: @@ -296,10 +375,13 @@ def readAndSetInputPathOnValidation(self): if not Path(self.reconstruction_input_data_loc.value).exists(): self.reconstruction_input_data_loc.value = self.input_directory return + result = self.reconstruction_input_data_loc.value self.directory = result self.current_dir_path = result - self.input_directory = result + self.input_directory = result + + self.saveLastPaths() # Copied from main_widget # ToDo: utilize common functions @@ -312,6 +394,8 @@ def browse_dir_path_output(self, elem): self.save_directory = result elem.value = self.save_directory + self.saveLastPaths() + # call back for output LineEdit path changed manually def readAndSetOutputPathOnValidation(self, elem): if elem.value is None or len(elem.value) == 0: @@ -321,6 +405,8 @@ def readAndSetOutputPathOnValidation(self, elem): self.directory = result self.save_directory = result + self.saveLastPaths() + # Copied from main_widget # ToDo: utilize common functions # Output data selector @@ -333,6 +419,8 @@ def browse_dir_path_model(self, elem): self.directory = self.model_directory self.current_dir_path = self.model_directory + self.saveLastPaths() + pydantic_models = list() for result in results: self.yaml_model_file = result @@ -372,51 +460,85 @@ def browse_dir_path_model(self, elem): return pydantic_models + # useful when using close widget and not napari close and we might need them again + def saveLastPaths(self): + HAS_INSTANCE["current_dir_path"] = self.current_dir_path + HAS_INSTANCE["current_save_path"] = self.current_save_path + HAS_INSTANCE["input_directory"] = self.input_directory + HAS_INSTANCE["save_directory"] = self.save_directory + HAS_INSTANCE["model_directory"] = self.model_directory + HAS_INSTANCE["yaml_model_file"] = self.yaml_model_file + + # clears the results table + def clear_results_table(self): + if self.confirmDialog(): + for i in range(self.proc_table_QFormLayout.count()): + self.proc_table_QFormLayout.removeRow(0) + # marks fields on the Model that cause a validation error - def modelHighlighter(self, errs, container:Container): + def modelHighlighter(self, errs): try: - self.modelHighlighterVals = {} - containerID = container.name - self.modelHighlighterVals[containerID] = {} - - errsList = list() - for err in errs: - errsList.append({"loc": err["loc"], "tooltip": err["msg"]}) - - self.modelHighlighterVals[containerID]["errs"] = errsList - self.modelHighlighterVals[containerID]["items"] = [] - - self.modelHighlighterSetter(errsList, container, containerID) + for uid in errs.keys(): + self.modelHighlighterVals[uid] = {} + container = errs[uid]["cls"] + self.modelHighlighterVals[uid]["errs"] = errs[uid]["errs"] + self.modelHighlighterVals[uid]["items"] = [] + self.modelHighlighterVals[uid]["tooltip"] = [] + if len(errs[uid]["errs"]) > 0: + self.modelHighlighterSetter(errs[uid]["errs"], container, uid) except Exception as exc: print(exc.args) # more of a test feature - no need to show up + # format all model errors into a display format for napari error message box + def formatStringForErrorDisplay(self, errs): + try: + ret_str = "" + for uid in errs.keys(): + if len(errs[uid]["errs"]) > 0: + ret_str += errs[uid]["collapsibleBox"] + "\n" + for idx in range(len(errs[uid]["errs"])): + ret_str += f"{'>'.join(errs[uid]['errs'][idx]['loc'])}:\n{errs[uid]['errs'][idx]['msg']} \n" + ret_str += "\n" + return ret_str + except Exception as exc: + return ret_str + # recursively fix the container for highlighting - def modelHighlighterSetter(self, errs, container:Container, containerID): + def modelHighlighterSetter(self, errs, container:Container, containerID, lev=0): try: layout = container.native.layout() for i in range(layout.count()): item = layout.itemAt(i) if item.widget(): widget = layout.itemAt(i).widget() - if not isinstance(widget._magic_widget, CheckBox) and isinstance(widget._magic_widget._inner_widget, Container) and not (widget._magic_widget._inner_widget is None): - self.modelHighlighterSetter(errs, widget._magic_widget._inner_widget, containerID) + if (not isinstance(widget._magic_widget, CheckBox) and not isinstance(widget._magic_widget, PushButton)) and not isinstance(widget._magic_widget, LineEdit) and isinstance(widget._magic_widget._inner_widget, Container) and not (widget._magic_widget._inner_widget is None): + self.modelHighlighterSetter(errs, widget._magic_widget._inner_widget, containerID, lev+1) else: - for err in errs: - if isinstance(widget._magic_widget, CheckBox): - if widget._magic_widget.label == err["loc"][len(err["loc"])-1].replace("_", " "): - widget._magic_widget.tooltip = err["tooltip"] - widget._magic_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") - self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget) - return - elif widget._magic_widget._label_widget.value == err["loc"][len(err["loc"])-1].replace("_", " "): - widget._magic_widget._label_widget.tooltip = err["tooltip"] - widget._magic_widget._label_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") - self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget._label_widget) - widget._magic_widget._inner_widget.tooltip = err["tooltip"] - widget._magic_widget._inner_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") - self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget._inner_widget) - return + for idx in range(len(errs)): + if len(errs[idx]["loc"])-1 < lev: + pass + elif isinstance(widget._magic_widget, CheckBox) or isinstance(widget._magic_widget, LineEdit) or isinstance(widget._magic_widget, PushButton): + if widget._magic_widget.label == errs[idx]["loc"][lev].replace("_", " "): + if widget._magic_widget.tooltip is None: + widget._magic_widget.tooltip = "-\n" + self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget) + self.modelHighlighterVals[containerID]["tooltip"].append(widget._magic_widget.tooltip) + widget._magic_widget.tooltip += errs[idx]["msg"] + "\n" + widget._magic_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") + elif widget._magic_widget._label_widget.value == errs[idx]["loc"][lev].replace("_", " "): + if widget._magic_widget._label_widget.tooltip is None: + widget._magic_widget._label_widget.tooltip = "-\n" + self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget._label_widget) + self.modelHighlighterVals[containerID]["tooltip"].append(widget._magic_widget._label_widget.tooltip) + widget._magic_widget._label_widget.tooltip += errs[idx]["msg"] + "\n" + widget._magic_widget._label_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") + if widget._magic_widget._inner_widget.tooltip is None: + widget._magic_widget._inner_widget.tooltip = "-\n" + self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget._inner_widget) + self.modelHighlighterVals[containerID]["tooltip"].append(widget._magic_widget._inner_widget.tooltip) + widget._magic_widget._inner_widget.tooltip += errs[idx]["msg"] + "\n" + widget._magic_widget._inner_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") except Exception as exc: print(exc.args) # more of a test feature - no need to show up @@ -426,9 +548,17 @@ def modelResetHighlighterSetter(self): try: for containerID in self.modelHighlighterVals.keys(): items = self.modelHighlighterVals[containerID]["items"] + tooltip = self.modelHighlighterVals[containerID]["tooltip"] + i=0 for widItem in items: # widItem.tooltip = None # let them tool tip remain widItem.native.setStyleSheet("border:1px solid rgb(0, 0, 0); border-width: 0px;") + widItem.tooltip = tooltip[i] + i += 1 + + except Exception as exc: + print(exc.args) + # more of a test feature - no need to show up except Exception as exc: print(exc.args) @@ -472,6 +602,7 @@ def addTableEntry(self, tableEntryID, tableEntryShortDesc, tableEntryVals, proc_ _scrollAreaCollapsibleBox = QScrollArea() _scrollAreaCollapsibleBox.setWidgetResizable(True) + _scrollAreaCollapsibleBox.setMinimumHeight(300) _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) _collapsibleBoxWidgetLayout = QVBoxLayout() @@ -535,8 +666,8 @@ def buildModel(self, selected_modes): model = None try: model = settings.ReconstructionSettings(input_channel_names=chNames, birefringence=b, phase=p, fluorescence=f) - except pydantic.v1.ValidationError as exc: - # use v1 for ValidationError - newer one is not caught properly + except ValidationError as exc: + # use v1 and v2 differ for ValidationError - newer one is not caught properly return None, exc.errors() model = self._fix_model(model, exclude_modes, 'input_channel_names', chNames) @@ -617,7 +748,14 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None self.messageBox(ret_msg) return pydantic_kwargs["time_indices"] = time_indices - + + if "birefringence" in pydantic_kwargs.keys(): + background_path, ret_msg = self.clean_path_string_when_empty("background_path", pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"]) + if background_path is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"] = background_path + # validate and return errors if None pydantic_model, ret_msg = self.validate_pydantic_model(pydantic_class, pydantic_kwargs) if pydantic_model is None: @@ -630,10 +768,7 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None if json_txt is None: self.messageBox(ret_msg) return - - # Add this container to the main scrollable widget - self.recon_tab_qwidget_settings_layout.addWidget(recon_pydantic_container.native) - + # Line seperator between pydantic UI components _line = QFrame() _line.setMinimumWidth(1) @@ -666,8 +801,17 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None _output_data_loc.changed.connect(lambda: self.readAndSetOutputPathOnValidation(_output_data_loc)) # Passing all UI components that would be deleted - _del_button.clicked.connect(lambda: self._delete_model(recon_pydantic_container.native, _output_data_loc.native, _output_data_btn.native, _del_button.native, _line, _idx, _str)) + _expandingTabEntryWidget = QWidget() + _del_button.clicked.connect(lambda: self._delete_model(_expandingTabEntryWidget, recon_pydantic_container.native, _output_data_loc.native, _output_data_btn.native, _del_button.native, _line, _idx, _str)) + c_mode = "-and-".join(selected_modes) + if c_mode in CONTAINERS_INFO.keys(): + CONTAINERS_INFO[c_mode] += 1 + else: + CONTAINERS_INFO[c_mode] = 1 + num_str = "{:02d}".format(CONTAINERS_INFO[c_mode]) + c_mode_str = f"{c_mode} - {num_str}" + # HBox for Output Data _hBox_widget = QWidget() _hBox_layout = QHBoxLayout() @@ -676,18 +820,49 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None _hBox_layout.addWidget(_output_data_loc.native) _hBox_layout.addWidget(_output_data_btn.native) - self.recon_tab_qwidget_settings_layout.addWidget(_hBox_widget) - self.recon_tab_qwidget_settings_layout.addWidget(_del_button.native) - self.recon_tab_qwidget_settings_layout.addWidget(_line) + # Add this container to the main scrollable widget + _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() + _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(Qt.AlignTop) + + _scrollAreaCollapsibleBoxWidget = QWidget() + _scrollAreaCollapsibleBoxWidget.setLayout(_scrollAreaCollapsibleBoxWidgetLayout) + + _scrollAreaCollapsibleBox = QScrollArea() + _scrollAreaCollapsibleBox.setWidgetResizable(True) + _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) + _scrollAreaCollapsibleBox.setMinimumHeight(500) + + _collapsibleBoxWidgetLayout = QVBoxLayout() + _collapsibleBoxWidgetLayout.setContentsMargins(0,0,0,0) + _collapsibleBoxWidgetLayout.setSpacing(0) + _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) - # Dynamic/modifying UI probably needs this - self.recon_tab_qwidget_settings_layout.addStretch() + _collapsibleBoxWidget = CollapsibleBox(c_mode_str) # tableEntryID, tableEntryShortDesc - should update with processing status + _collapsibleBoxWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) + + _expandingTabEntryWidgetLayout = QVBoxLayout() + _expandingTabEntryWidgetLayout.addWidget(_collapsibleBoxWidget) + + _expandingTabEntryWidget.toolTip = c_mode_str + _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) + _expandingTabEntryWidget.layout().setContentsMargins(0,0,0,0) + _expandingTabEntryWidget.layout().setSpacing(0) + _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + _expandingTabEntryWidget.layout().setAlignment(Qt.AlignTop) + + _scrollAreaCollapsibleBoxWidgetLayout.addWidget(recon_pydantic_container.native) + _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget) + _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_del_button.native) + _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_line) + + self.recon_tab_qwidget_settings_layout.addWidget(_expandingTabEntryWidget) # Store a copy of the pydantic container along with all its associated components and properties # We dont needs a copy of the class but storing for now # This will be used for making deletion edits and looping to create our final run output # uuid - used for identiying in editable list - self.pydantic_classes.append({'uuid':_str, 'class':pydantic_class, 'input':self.reconstruction_input_data_loc, 'output':_output_data_loc, 'container':recon_pydantic_container, 'selected_modes':selected_modes.copy(), 'exclude_modes':exclude_modes.copy()}) + self.pydantic_classes.append({'uuid':_str, 'c_mode_str':c_mode_str, 'collapsibleBoxWidget':_collapsibleBoxWidget, 'class':pydantic_class, 'input':self.reconstruction_input_data_loc, 'output':_output_data_loc, 'container':recon_pydantic_container, 'selected_modes':selected_modes.copy(), 'exclude_modes':exclude_modes.copy()}) self.index += 1 if self.index > 1: @@ -698,8 +873,11 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None return pydantic_model # UI components deletion - maybe just needs the parent container instead of individual components - def _delete_model(self, wid1, wid2, wid3, wid4, wid5, index, _str): + def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, index, _str): + if not self.confirmDialog(): + return False + if wid5 is not None: wid5.setParent(None) if wid4 is not None: @@ -710,15 +888,16 @@ def _delete_model(self, wid1, wid2, wid3, wid4, wid5, index, _str): wid2.setParent(None) if wid1 is not None: wid1.setParent(None) + if wid0 is not None: + wid0.setParent(None) # Find and remove the class from our pydantic model list using uuid i=0 for item in self.pydantic_classes: if item["uuid"] == _str: self.pydantic_classes.pop(i) - return - i += 1 - self.index = i + break + self.index = len(self.pydantic_classes) if self.index > 1: self.build_button.text = "Build && Run {n} Models".format(n=self.index) else: @@ -726,15 +905,17 @@ def _delete_model(self, wid1, wid2, wid3, wid4, wid5, index, _str): # Clear all the generated pydantic models and clears the pydantic model list def _clear_all_models(self): - index = self.recon_tab_qwidget_settings_layout.count()-1 - while(index >= 0): - myWidget = self.recon_tab_qwidget_settings_layout.itemAt(index).widget() - if myWidget is not None: - myWidget.setParent(None) - index -=1 - self.pydantic_classes.clear() - self.index = 0 - self.build_button.text = "Build && Run Model" + if self.confirmDialog(): + index = self.recon_tab_qwidget_settings_layout.count()-1 + while(index >= 0): + myWidget = self.recon_tab_qwidget_settings_layout.itemAt(index).widget() + if myWidget is not None: + myWidget.setParent(None) + index -=1 + self.pydantic_classes.clear() + CONTAINERS_INFO.clear() + self.index = 0 + self.build_button.text = "Build && Run Model" # Displays the json output from the pydantic model UI selections by user # Loops through all our stored pydantic classes @@ -745,12 +926,21 @@ def build_model_and_run(self): # second pass for creating yaml and processing self.modelResetHighlighterSetter() # reset the container elements that might be highlighted for errors - + _collectAllErrors = {} + _collectAllErrorsBool = True for item in self.pydantic_classes: cls = item['class'] cls_container = item['container'] selected_modes = item['selected_modes'] - exclude_modes = item['exclude_modes'] + exclude_modes = item['exclude_modes'] + uuid_str = item['uuid'] + _collapsibleBoxWidget = item['collapsibleBoxWidget'] + c_mode_str = item['c_mode_str'] + + _collectAllErrors[uuid_str] = {} + _collectAllErrors[uuid_str]["cls"] = cls_container + _collectAllErrors[uuid_str]["errs"] = [] + _collectAllErrors[uuid_str]["collapsibleBox"] = c_mode_str # build up the arguments for the pydantic model given the current container if cls is None: @@ -759,38 +949,57 @@ def build_model_and_run(self): # get the kwargs from the container/class pydantic_kwargs = {} - pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args(cls_container, cls, pydantic_kwargs, exclude_modes) - if pydantic_kwargs is None: + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args(cls_container, cls, pydantic_kwargs, exclude_modes) + if pydantic_kwargs is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return # For list element, this needs to be cleaned and parsed back as an array input_channel_names, ret_msg = self.clean_string_for_list("input_channel_names", pydantic_kwargs["input_channel_names"]) - if input_channel_names is None: + if input_channel_names is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return pydantic_kwargs["input_channel_names"] = input_channel_names time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) - if time_indices is None: + if time_indices is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return pydantic_kwargs["time_indices"] = time_indices + if "birefringence" in pydantic_kwargs.keys(): + background_path, ret_msg = self.clean_path_string_when_empty("background_path", pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"]) + if background_path is None and not _collectAllErrorsBool: + self.messageBox(ret_msg) + return + pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"] = background_path + # validate and return errors if None # ToDo: error causing fields could be marked red pydantic_model, ret_msg = self.validate_pydantic_model(cls, pydantic_kwargs) - if pydantic_model is None: - self.modelHighlighter(ret_msg, cls_container) + if ret_msg == MSG_SUCCESS: + _collapsibleBoxWidget.setNewName(f"{c_mode_str} {_validate_ok}") + else: + _collapsibleBoxWidget.setNewName(f"{c_mode_str} {_validate_alert}") + _collectAllErrors[uuid_str]["errs"] = ret_msg + if pydantic_model is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return # generate a json from the instantiated model, update the json_display # most of this will end up in a table as processing proceeds json_txt, ret_msg = self.validate_and_return_json(pydantic_model) - if json_txt is None: + if json_txt is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return + + # check if we collected any validation errors before continuing + for uu_key in _collectAllErrors.keys(): + if len(_collectAllErrors[uu_key]["errs"]) > 0: + self.modelHighlighter(_collectAllErrors) + fmt_str = self.formatStringForErrorDisplay(_collectAllErrors) + self.messageBox(fmt_str) + return # generate a time-stamp for our yaml files to avoid overwriting # files generated at the same time will have an index suffix @@ -804,6 +1013,7 @@ def build_model_and_run(self): cls_container = item['container'] selected_modes = item['selected_modes'] exclude_modes = item['exclude_modes'] + c_mode_str = item['c_mode_str'] # gather input/out locations input_dir = f"{item['input'].value}" @@ -832,6 +1042,19 @@ def build_model_and_run(self): return pydantic_kwargs["time_indices"] = time_indices + time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) + if time_indices is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["time_indices"] = time_indices + + if "birefringence" in pydantic_kwargs.keys(): + background_path, ret_msg = self.clean_path_string_when_empty("background_path", pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"]) + if background_path is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"] = background_path + # validate and return errors if None pydantic_model, ret_msg = self.validate_pydantic_model(cls, pydantic_kwargs) if pydantic_model is None: @@ -848,11 +1071,10 @@ def build_model_and_run(self): # save the yaml files # ToDo: error catching and validation for path # path selection ??? - save_config_path = str(Path.cwd()) - dir_ = save_config_path + save_config_path = str(Path(output_dir).parent.absolute()) yml_file_name = "-and-".join(selected_modes) yml_file = yml_file_name+"-"+unique_id+"-"+str(i)+".yml" - config_path = os.path.join(dir_ ,"examples", yml_file) + config_path = os.path.join(save_config_path, yml_file) utils.model_to_yaml(pydantic_model, config_path) # Input params for table entry @@ -865,7 +1087,7 @@ def build_model_and_run(self): # addl_txt = "ID:" + unique_id + "-"+ str(i) + "\nInput:" + input_dir + "\nOutput:" + output_dir # self.json_display.value = self.json_display.value + addl_txt + "\n" + json_txt+ "\n\n" expID = "{tID}-{idx}".format(tID = unique_id, idx = i) - tableID = "{tName}: ({tID}-{idx})".format(tName = yml_file_name, tID = unique_id, idx = i) + tableID = "{tName}: ({tID}-{idx})".format(tName = c_mode_str, tID = unique_id, idx = i) tableDescToolTip = "{tName}: ({tID}-{idx})".format(tName = yml_file_name, tID = unique_id, idx = i) proc_params = {} @@ -878,6 +1100,11 @@ def build_model_and_run(self): self.addTableEntry(tableID, tableDescToolTip, json_txt, proc_params) + # ======= These function do not implement validation + # They simply make the data from GUI translate to input types + # that the model expects: for eg. GUI txt field will output only str + # when the model needs integers + # util function to parse list elements displayed as string def remove_chars(self, string, chars_to_remove): for char in chars_to_remove: @@ -903,15 +1130,14 @@ def clean_string_for_list(self, field, string): # [1,2,3], 4,5,6 , 5-95 def clean_string_int_for_list(self, field, string): chars_to_remove = ['[',']', '\'', '"', ' '] + if Literal[string] == Literal["all"]: + return string, MSG_SUCCESS + if Literal[string] == Literal[""]: + return string, MSG_SUCCESS if isinstance(string, str): string = self.remove_chars(string, chars_to_remove) if len(string) == 0: - return None, {'msg':field + ' is invalid'} - if 'all' in string: - if Literal[string] == Literal["all"]: - return string, MSG_SUCCESS - else: - return None, {'msg':field + ' can only contain \'all\' as string field'} + return None, {'msg':field + ' is invalid'} if '-' in string: string = string.split('-') if len(string) == 2: @@ -937,6 +1163,13 @@ def clean_string_int_for_list(self, field, string): string = string.split(',') return string, MSG_SUCCESS return string, MSG_SUCCESS + + # util function to set path to empty - by default empty path has a "." + def clean_path_string_when_empty(self, field, string): + if isinstance(string, Path) and string == Path(""): + string = "" + return string, MSG_SUCCESS + return string, MSG_SUCCESS # get the pydantic_kwargs and catches any errors in doing so def get_and_validate_pydantic_args(self, cls_container, cls, pydantic_kwargs, exclude_modes): @@ -944,7 +1177,7 @@ def get_and_validate_pydantic_args(self, cls_container, cls, pydantic_kwargs, ex try: self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs, exclude_modes) return pydantic_kwargs, MSG_SUCCESS - except pydantic.v1.ValidationError as exc: + except ValidationError as exc: return None, exc.errors() except Exception as exc: return None, exc.args @@ -956,7 +1189,7 @@ def validate_pydantic_model(self, cls, pydantic_kwargs): try : pydantic_model = settings.ReconstructionSettings.parse_obj(pydantic_kwargs) return pydantic_model, MSG_SUCCESS - except pydantic.v1.ValidationError as exc: + except ValidationError as exc: return None, exc.errors() except Exception as exc: return None, exc.args @@ -976,7 +1209,7 @@ def get_model_from_file(self, model_file_path): try : pydantic_model = utils.yaml_to_model(model_file_path, settings.ReconstructionSettings) if pydantic_model is None: - raise Exception("yaml_to_model - returned a None model") + raise Exception("utils.yaml_to_model - returned a None model") return pydantic_model, MSG_SUCCESS except Exception as exc: return None, exc.args @@ -1002,54 +1235,72 @@ def convert(self, obj): # excludes handles fields that are not supposed to show up from __fields__ # json_dict adds ability to provide new set of default values at time of container creation - def add_pydantic_to_container(self, py_model:Union[pydantic.BaseModel, ModelMetaclass], container: widgets.Container, excludes=[], json_dict=None): + def add_pydantic_to_container(self, py_model:Union[BaseModel, ModelMetaclass], container: widgets.Container, excludes=[], json_dict=None): # recursively traverse a pydantic model adding widgets to a container. When a nested # pydantic model is encountered, add a new nested container for field, field_def in py_model.__fields__.items(): if field_def is not None and field not in excludes: def_val = field_def.default - ftype = field_def.type_ - if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, ModelMetaclass) or isinstance(ftype, ModelMetaclass): + ftype = field_def.type_ + toolTip = "" + try: + for f_val in field_def.class_validators.keys(): + toolTip = f"{toolTip}{f_val} " + except Exception as e: + pass + if isinstance(ftype, BaseModel) or isinstance(ftype, ModelMetaclass): json_val = None if json_dict is not None: json_val = json_dict[field] # the field is a pydantic class, add a container for it and fill it new_widget_cls = widgets.Container new_widget = new_widget_cls(name=field_def.name) + new_widget.tooltip = toolTip self.add_pydantic_to_container(ftype, new_widget, excludes, json_val) #ToDo: Implement Union check, tried: # pydantic.typing.is_union(ftype) # isinstance(ftype, types.UnionType) # https://stackoverflow.com/questions/45957615/how-to-check-a-variable-against-union-type-during-runtime - elif isinstance(def_val, str): #field == "time_indices": - new_widget_cls, ops = get_widget_class(None, str, dict(name=field, value=def_val)) - new_widget = new_widget_cls(**ops) - new_widget.tooltip = field + elif isinstance(ftype, type(Union[NonNegativeInt, List, str])): + if (field == "background_path"): #field == "background_path": + new_widget_cls, ops = get_widget_class(def_val, Annotated[Path, {"mode": "d"}], dict(name=field, value=def_val)) + new_widget = new_widget_cls(**ops) + elif (field == "time_indices"): #field == "time_indices": + new_widget_cls, ops = get_widget_class(def_val, str, dict(name=field, value=def_val)) + new_widget = new_widget_cls(**ops) + else: # other Union cases + new_widget_cls, ops = get_widget_class(def_val, str, dict(name=field, value=def_val)) + new_widget = new_widget_cls(**ops) + new_widget.tooltip = toolTip if isinstance(new_widget, widgets.EmptyWidget): - warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") elif isinstance(def_val, float): # parse the field, add appropriate widget + def_step_size = 0.001 + if field_def.name == "regularization_strength": + def_step_size = 0.00001 if def_val > -1 and def_val < 1: - new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val, step=float(0.001))) + new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val, step=float(def_step_size))) new_widget = new_widget_cls(**ops) - new_widget.tooltip = field_def.name + new_widget.tooltip = toolTip else: new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val)) new_widget = new_widget_cls(**ops) - new_widget.tooltip = field_def.name + new_widget.tooltip = toolTip if isinstance(new_widget, widgets.EmptyWidget): warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") else: # parse the field, add appropriate widget new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val)) - new_widget = new_widget_cls(**ops) - new_widget.tooltip = field_def.name + new_widget = new_widget_cls(**ops) if isinstance(new_widget, widgets.EmptyWidget): warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + else: + new_widget.tooltip = toolTip if json_dict is not None and not isinstance(new_widget, widgets.Container): if isinstance(new_widget, widgets.CheckBox): - new_widget.value = True if json_dict[field]=="true" else False + new_widget.value = True if json_dict[field]=="true" else False else: new_widget.value = json_dict[field] container.append(new_widget) @@ -1063,7 +1314,7 @@ def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pyda for field, field_def in pydantic_model.__fields__.items(): if field_def is not None and field not in excludes: ftype = field_def.type_ - if isinstance(ftype, pydantic.BaseModel) or isinstance(ftype, ModelMetaclass) or isinstance(ftype, ModelMetaclass): + if isinstance(ftype, BaseModel) or isinstance(ftype, ModelMetaclass): # go deeper pydantic_kwargs[field] = {} # new dictionary for the new nest level # any pydantic class will be a container, so pull that out to pass @@ -1165,6 +1416,9 @@ def __init__(self, title="", parent=None): QtCore.QPropertyAnimation(self.content_area, b"maximumHeight") ) + def setNewName(self, name): + self.toggle_button.setText(name) + # @QtCore.pyqtSlot() def on_pressed(self): checked = self.toggle_button.isChecked() @@ -1202,9 +1456,10 @@ def setContentLayout(self, layout): import socket, threading class MyWorker(): - def __init__(self, formLayout): + def __init__(self, formLayout, parentForm): super().__init__() - self.formLayout = formLayout + self.formLayout:QFormLayout = formLayout + self.ui:QWidget = parentForm self.max_cores = os.cpu_count() # In the case of CLI, we just need to submit requests in a non-blocking way self.threadPool = int(self.max_cores/2) @@ -1222,6 +1477,10 @@ def __init__(self, formLayout): thread = threading.Thread(target=self.startServer) thread.start() + def setNewInstances(self, formLayout, parentForm): + self.formLayout:QFormLayout = formLayout + self.ui:QWidget = parentForm + def findWidgetRowInLayout(self, strID): layout: QFormLayout = self.formLayout for idx in range(0, layout.rowCount()): @@ -1242,7 +1501,7 @@ def removeRow(self, row, expID): name_widget = widgetItem.widget() toolTip_string = str(name_widget.toolTip) if expID in toolTip_string: - layout.removeRow(row) + layout.takeRow(row) # removeRow vs takeRow for threads ? except Exception as exc: print(exc.args) @@ -1257,6 +1516,8 @@ def startServer(self): while self.serverRunning: connection, address = self.serverSocket.accept() + if not self.ui.isVisible(): + break try: buf = connection.recv(64) if len(buf) > 0: @@ -1360,6 +1621,11 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): self.workerThread.start() time.sleep(2) # break - based on status + elif JOB_TRIGGERED_EXC in jobTXT: + self.results[expIdx]["status"] = STATUS_errored_job + jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") + _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobTXT +"\n\n"+ jobERR + break elif JOB_RUNNING_STR in jobTXT: self.results[expIdx]["status"] = STATUS_running_job _infoBox.value = jobTXT @@ -1372,12 +1638,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): else: _infoBox.value = "Please check terminal output for Job status..\n\n" + jobTXT if _tUpdateCount > _tUpdateCountTimeout: - break - elif JOB_TRIGGERED_EXC in jobTXT: - self.results[expIdx]["status"] = STATUS_errored_job - jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") - _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobTXT +"\n\n"+ jobERR - break + break else: jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobERR @@ -1402,7 +1663,7 @@ def runInPool(self, params): self.pool.submit(self.run, params) except Exception as exc: self.results[params["exp_id"]]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["error"] = str(exc.args) + self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() def runMultiInPool(self, multi_params_as_list): @@ -1415,7 +1676,7 @@ def runMultiInPool(self, multi_params_as_list): except Exception as exc: for params in multi_params_as_list: self.results[params["exp_id"]]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["error"] = str(exc.args) + self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() def getResults(self): @@ -1443,7 +1704,7 @@ def run(self, params): except Exception as exc: self.results[params["exp_id"]]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["error"] = str(exc.args) + self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() def runInSubProcess(self, params): @@ -1461,9 +1722,9 @@ def runInSubProcess(self, params): if proc.returncode != 0: raise Exception("An error occurred in processing ! Check terminal output.") - except Exception as exc: + except Exception as exc: self.results[params["exp_id"]]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["error"] = str(exc.args) + self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() # Emits a signal to QFormLayout on the main thread From d23b77f2546ca7edeb1e0bd6ab86d11823da3346 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Wed, 18 Dec 2024 16:37:22 -0500 Subject: [PATCH 08/38] Update setup.cfg - pinning pydantic lib ver to 1.10.19 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f36021c7..9ca61f10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ install_requires = wget>=3.2 psutil submitit - pydantic>=1.10.17 + pydantic==1.10.19 [options.extras_require] dev = From 3d542888f152a454270ada0168bcc5313b8a3794 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Wed, 18 Dec 2024 17:07:43 -0500 Subject: [PATCH 09/38] use PyQt6 for pyqtSignal use PyQt6 for pyqtSignal --- recOrder/plugin/tab_recon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index f75944e3..a6d4c585 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -5,7 +5,7 @@ from qtpy.QtCore import Qt, QEvent, QThread from qtpy.QtWidgets import * from magicgui.widgets import * -from PyQt5.QtCore import pyqtSignal +from PyQt6.QtCore import pyqtSignal from iohub.ngff import Plate, open_ome_zarr from natsort import natsorted From 29c6a984871c79d78e388c29c37f10a67ec7104b Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Thu, 19 Dec 2024 02:25:22 -0500 Subject: [PATCH 10/38] standalone - fixes standalone widget --- recOrder/cli/jobs_mgmt.py | 3 ++- recOrder/plugin/tab_recon.py | 4 ++-- .../widget_tests/test_pydantic_model_widget.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 recOrder/tests/widget_tests/test_pydantic_model_widget.py diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py index d8a1e628..b2047136 100644 --- a/recOrder/cli/jobs_mgmt.py +++ b/recOrder/cli/jobs_mgmt.py @@ -1,7 +1,8 @@ -import submitit, os, json, time +import os, json import socket from pathlib import Path from tempfile import TemporaryDirectory +import submitit # Jobs query object # Todo: Not sure where these should functions should reside - ask Talon diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index a6d4c585..8dbe33a5 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -302,7 +302,7 @@ def closeEvent(self, event): self.worker.stopServer() def hideEvent(self, event): - if event.type() == QEvent.Type.Hide and self._ui.isVisible(): + if event.type() == QEvent.Type.Hide and (self._ui is not None and self._ui.isVisible()): pass def showEvent(self, event): @@ -1516,7 +1516,7 @@ def startServer(self): while self.serverRunning: connection, address = self.serverSocket.accept() - if not self.ui.isVisible(): + if self.ui is not None and not self.ui.isVisible(): break try: buf = connection.recv(64) diff --git a/recOrder/tests/widget_tests/test_pydantic_model_widget.py b/recOrder/tests/widget_tests/test_pydantic_model_widget.py new file mode 100644 index 00000000..ffad88f5 --- /dev/null +++ b/recOrder/tests/widget_tests/test_pydantic_model_widget.py @@ -0,0 +1,18 @@ +import sys +from PyQt6.QtWidgets import QApplication, QWidget, QVBoxLayout + +from recOrder.plugin import tab_recon + +class MainWindow(QWidget): + def __init__(self): + super().__init__() + recon_tab = tab_recon.Ui_Form() + layout = QVBoxLayout() + self.setLayout(layout) + layout.addWidget(recon_tab.recon_tab_widget) + +if __name__ == "__main__": + app = QApplication(sys.argv) + window = MainWindow() + window.show() + sys.exit(app.exec_()) \ No newline at end of file From adf0b7b15518267442c55286135e9592cf24b0f9 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Fri, 20 Dec 2024 11:13:04 -0500 Subject: [PATCH 11/38] fixes for BG, etc - fixes BG selection and resetting if Path is invalid - fixes removal of row/stability - using Queued List to counter race conditions - added .zarr data validation check for Input, also used for BG - make uniqueID more unique - break thread update operations when user has cleared the results table - standalone GUI niceties --- recOrder/plugin/gui.py | 2 +- recOrder/plugin/tab_recon.py | 210 ++++++++++++------ .../test_pydantic_model_widget.py | 17 +- 3 files changed, 154 insertions(+), 75 deletions(-) diff --git a/recOrder/plugin/gui.py b/recOrder/plugin/gui.py index 9f03ccb3..c2f32797 100644 --- a/recOrder/plugin/gui.py +++ b/recOrder/plugin/gui.py @@ -925,7 +925,7 @@ def setupUi(self, Form): self.gridLayout_6.addWidget(self.scrollArea_4, 4, 0, 1, 1) self.tabWidget.addTab(self.Acquisition, "") - self.recon_tab = tab_recon.Ui_Form(Form) + self.recon_tab = tab_recon.Ui_ReconTab_Form(Form) self.tabWidget.addTab(self.recon_tab.recon_tab_widget, 'Reconstruction') self.Display = QtWidgets.QWidget() diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 8dbe33a5..9bc58be4 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -64,6 +64,7 @@ STATUS_finished_job = "Finished_Job" STATUS_errored_pool = "Errored_Pool" STATUS_errored_job = "Errored_Job" +STATUS_user_cleared_job = "User_Cleared_Job" MSG_SUCCESS = {'msg':'success'} JOB_COMPLETION_STR = "Job completed successfully" @@ -91,7 +92,7 @@ # Not efficient since instantiated from GUI # Does not have access to common functions in main_widget # ToDo : From main_widget and pass self reference -class Ui_Form(QWidget): +class Ui_ReconTab_Form(QWidget): def __init__(self, parent=None): super().__init__(parent) @@ -110,7 +111,10 @@ def __init__(self, parent=None): self.input_directory = str(Path.cwd()) self.save_directory = str(Path.cwd()) self.model_directory = str(Path.cwd()) - self.yaml_model_file = str(Path.cwd()) + self.yaml_model_file = str(Path.cwd()) + + self.input_directory_dataset = None + self.input_directory_datasetMeta = None # Top level parent self.recon_tab_widget = QWidget() @@ -282,9 +286,9 @@ def __init__(self, parent=None): # handle napari's close widget and avoid starting a second server if HAS_INSTANCE["val"]: self.worker:MyWorker = HAS_INSTANCE["MyWorker"] - self.worker.setNewInstances(self.proc_table_QFormLayout, self._ui) + self.worker.setNewInstances(self.proc_table_QFormLayout, self, self._ui) else: - self.worker = MyWorker(self.proc_table_QFormLayout, self._ui) + self.worker = MyWorker(self.proc_table_QFormLayout, self, self._ui) HAS_INSTANCE["val"] = True HAS_INSTANCE["MyWorker"] = self.worker @@ -350,19 +354,18 @@ def browse_dir_path_inputBG(self, elem): elem.value = result # not working - not used - def validateInputData(self, input_data_folder: str) -> list[Path]: + def validateInputData(self, input_data_folder: str, metadata=False) -> bool: # Sort and validate the input paths, expanding plates into lists of positions - return True, MSG_SUCCESS + # return True, MSG_SUCCESS try: - input_paths = [Path(path) for path in natsorted(input_data_folder)] - for path in input_paths: - with open_ome_zarr(path, mode="r") as dataset: - if isinstance(dataset, Plate): - plate_path = input_paths.pop() - for position in dataset.positions(): - input_paths.append(plate_path / position[0]) - - return True, MSG_SUCCESS + input_paths = Path(input_data_folder) + with open_ome_zarr(input_paths, mode="r") as dataset: + # ToDo: Metadata reading and implementation in GUI for + # channel names, time indicies, etc. + if metadata: + self.input_directory_dataset = dataset + return True, MSG_SUCCESS + raise Exception("Dataset does not appear to be a valid ome-zarr storage") except Exception as exc: return False, exc.args @@ -371,17 +374,25 @@ def validateInputData(self, input_data_folder: str) -> list[Path]: def readAndSetInputPathOnValidation(self): if self.reconstruction_input_data_loc.value is None or len(self.reconstruction_input_data_loc.value) == 0: self.reconstruction_input_data_loc.value = self.input_directory + self.messageBox("Input data path cannot be empty") return if not Path(self.reconstruction_input_data_loc.value).exists(): self.reconstruction_input_data_loc.value = self.input_directory + self.messageBox("Input data path must point to a valid location") return result = self.reconstruction_input_data_loc.value - self.directory = result - self.current_dir_path = result - self.input_directory = result + valid, ret_msg = self.validateInputData(result) - self.saveLastPaths() + if valid: + self.directory = result + self.current_dir_path = result + self.input_directory = result + + self.saveLastPaths() + else: + self.reconstruction_input_data_loc.value = self.input_directory + self.messageBox(ret_msg) # Copied from main_widget # ToDo: utilize common functions @@ -448,8 +459,27 @@ def browse_dir_path_model(self, elem): pydantic_model, ret_msg = self.get_model_from_file(self.yaml_model_file) if pydantic_model is None: - self.messageBox(ret_msg) - return + if isinstance(ret_msg, List) and len(ret_msg)==2 and len(ret_msg[0]["loc"])==3 and ret_msg[0]["loc"][2] == "background_path": + pydantic_model = pruned_pydantic_class # if only background_path fails validation + json_dict["birefringence"]["apply_inverse"]["background_path"] = "" + self.messageBox("background_path:\nPath was invalid and will be reset") + else: + self.messageBox(ret_msg) + return + else: + # make sure "background_path" is valid + bg_loc = json_dict["birefringence"]["apply_inverse"]["background_path"] + if bg_loc != "": + extension = os.path.splitext(bg_loc)[1] + if len(extension) > 0: + bg_loc = Path(os.path.join(str(Path(bg_loc).parent.absolute()),"background.zarr")) + else: + bg_loc = Path(os.path.join(bg_loc, "background.zarr")) + if not bg_loc.exists() or not self.validateInputData(str(bg_loc)): + self.messageBox("background_path:\nPwas invalid and will be reset") + json_dict["birefringence"]["apply_inverse"]["background_path"] = "" + else: + json_dict["birefringence"]["apply_inverse"]["background_path"] = str(bg_loc.parent.absolute()) pydantic_model = self._create_acq_contols2(selected_modes, exclude_modes, pydantic_model, json_dict) if pydantic_model is None: @@ -472,8 +502,20 @@ def saveLastPaths(self): # clears the results table def clear_results_table(self): if self.confirmDialog(): - for i in range(self.proc_table_QFormLayout.count()): + for i in range(self.proc_table_QFormLayout.rowCount()): self.proc_table_QFormLayout.removeRow(0) + + def removeRow(self, row, expID): + try: + if row < self.proc_table_QFormLayout.rowCount(): + widgetItem = self.proc_table_QFormLayout.itemAt(row) + if widgetItem is not None: + name_widget = widgetItem.widget() + toolTip_string = str(name_widget.toolTip) + if expID in toolTip_string: + self.proc_table_QFormLayout.removeRow(row) # removeRow vs takeRow for threads ? + except Exception as exc: + print(exc.args) # marks fields on the Model that cause a validation error def modelHighlighter(self, errs): @@ -602,7 +644,7 @@ def addTableEntry(self, tableEntryID, tableEntryShortDesc, tableEntryVals, proc_ _scrollAreaCollapsibleBox = QScrollArea() _scrollAreaCollapsibleBox.setWidgetResizable(True) - _scrollAreaCollapsibleBox.setMinimumHeight(300) + _scrollAreaCollapsibleBox.setMinimumHeight(200) _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) _collapsibleBoxWidgetLayout = QVBoxLayout() @@ -975,7 +1017,6 @@ def build_model_and_run(self): pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"] = background_path # validate and return errors if None - # ToDo: error causing fields could be marked red pydantic_model, ret_msg = self.validate_pydantic_model(cls, pydantic_kwargs) if ret_msg == MSG_SUCCESS: _collapsibleBoxWidget.setNewName(f"{c_mode_str} {_validate_ok}") @@ -1004,7 +1045,8 @@ def build_model_and_run(self): # generate a time-stamp for our yaml files to avoid overwriting # files generated at the same time will have an index suffix now = datetime.datetime.now() - unique_id = now.strftime("%Y_%m_%d_%H_%M_%S") + ms = now.strftime("%f")[:3] + unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_")+ms i = 0 for item in self.pydantic_classes: @@ -1069,8 +1111,7 @@ def build_model_and_run(self): return # save the yaml files - # ToDo: error catching and validation for path - # path selection ??? + # path is next to saved data location save_config_path = str(Path(output_dir).parent.absolute()) yml_file_name = "-and-".join(selected_modes) yml_file = yml_file_name+"-"+unique_id+"-"+str(i)+".yml" @@ -1206,8 +1247,12 @@ def validate_and_return_json(self, pydantic_model): # will get all fields (even those that are optional and not in yaml) and default values # model needs further parsing against yaml file for fields def get_model_from_file(self, model_file_path): + pydantic_model = None try : - pydantic_model = utils.yaml_to_model(model_file_path, settings.ReconstructionSettings) + try: + pydantic_model = utils.yaml_to_model(model_file_path, settings.ReconstructionSettings) + except ValidationError as exc: + return pydantic_model, exc.errors() if pydantic_model is None: raise Exception("utils.yaml_to_model - returned a None model") return pydantic_model, MSG_SUCCESS @@ -1266,6 +1311,7 @@ def add_pydantic_to_container(self, py_model:Union[BaseModel, ModelMetaclass], c if (field == "background_path"): #field == "background_path": new_widget_cls, ops = get_widget_class(def_val, Annotated[Path, {"mode": "d"}], dict(name=field, value=def_val)) new_widget = new_widget_cls(**ops) + toolTip = "Select the folder containing background.zarr" elif (field == "time_indices"): #field == "time_indices": new_widget_cls, ops = get_widget_class(def_val, str, dict(name=field, value=def_val)) new_widget = new_widget_cls(**ops) @@ -1298,11 +1344,19 @@ def add_pydantic_to_container(self, py_model:Union[BaseModel, ModelMetaclass], c warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") else: new_widget.tooltip = toolTip - if json_dict is not None and not isinstance(new_widget, widgets.Container): - if isinstance(new_widget, widgets.CheckBox): - new_widget.value = True if json_dict[field]=="true" else False - else: - new_widget.value = json_dict[field] + if json_dict is not None and (not isinstance(new_widget, widgets.Container) or (isinstance(new_widget, widgets.FileEdit))): + if field in json_dict.keys(): + if isinstance(new_widget, widgets.CheckBox): + new_widget.value = True if json_dict[field]=="true" else False + elif isinstance(new_widget, widgets.FileEdit): + if len(json_dict[field]) > 0: + extension = os.path.splitext(json_dict[field])[1] + if len(extension) > 0: + new_widget.value = Path(json_dict[field]).parent.absolute() # CLI accepts BG folder not .zarr + else: + new_widget.value = Path(json_dict[field]) + else: + new_widget.value = json_dict[field] container.append(new_widget) # refer - add_pydantic_to_container() for comments @@ -1456,9 +1510,10 @@ def setContentLayout(self, layout): import socket, threading class MyWorker(): - def __init__(self, formLayout, parentForm): + def __init__(self, formLayout, tab_recon:Ui_ReconTab_Form, parentForm): super().__init__() self.formLayout:QFormLayout = formLayout + self.tab_recon:Ui_ReconTab_Form = tab_recon self.ui:QWidget = parentForm self.max_cores = os.cpu_count() # In the case of CLI, we just need to submit requests in a non-blocking way @@ -1469,17 +1524,21 @@ def __init__(self, formLayout, parentForm): # self.runner = CliRunner() self.startPool() # jobs_mgmt.shared_var_jobs = self.JobsManager.shared_var_jobs - self.lock = threading.Lock() self.JobsMgmt = jobs_mgmt.JobsManagement() self.useServer = True self.serverRunning = True self.serverSocket = None thread = threading.Thread(target=self.startServer) thread.start() + self.workerThreadRowDeletion = RowDeletionWorkerThread(self.formLayout) + self.workerThreadRowDeletion.removeRowSignal.connect(self.tab_recon.removeRow) + self.workerThreadRowDeletion.start() - def setNewInstances(self, formLayout, parentForm): + def setNewInstances(self, formLayout, tab_recon, parentForm): self.formLayout:QFormLayout = formLayout + self.tab_recon:Ui_ReconTab_Form = tab_recon self.ui:QWidget = parentForm + self.workerThreadRowDeletion.setNewInstances(formLayout) def findWidgetRowInLayout(self, strID): layout: QFormLayout = self.formLayout @@ -1492,19 +1551,6 @@ def findWidgetRowInLayout(self, strID): return idx return -1 - def removeRow(self, row, expID): - try: - with self.lock: - if row < self.formLayout.rowCount(): - layout: QFormLayout = self.formLayout - widgetItem = layout.itemAt(row) - name_widget = widgetItem.widget() - toolTip_string = str(name_widget.toolTip) - if expID in toolTip_string: - layout.takeRow(row) # removeRow vs takeRow for threads ? - except Exception as exc: - print(exc.args) - def startServer(self): try: if not self.useServer: @@ -1574,20 +1620,36 @@ def shutDownPool(self): def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): # finished will be updated by the job - submitit status + # ToDo: Another approach to this could be to implement a status thread on the client side + # Since the client is already running till the job is completed, the client could ping status + # at regular intervals and also provide results and exceptions we currently read from the file + # Currently we only send JobID/UniqueID pair from Client to Server. This would reduce multiple threads + # server side. + # For row removal use a Queued list approach for better stability + if expIdx != "" and jobIdx != "": # this request came from server so we can wait for the Job to finish and update progress # some wait logic needs to be added otherwise for unknown errors this thread will persist # perhaps set a time out limit and then update the status window and then exit params = self.results[expIdx] - _infoBox = params["table_entry_infoBox"] + _infoBox:Label = params["table_entry_infoBox"] _txtForInfoBox = "Updating {id}: Please wait... \nJobID assigned: {jID} ".format(id=params["desc"], jID=jobIdx) - _infoBox.value = _txtForInfoBox + try: + _infoBox.value = _txtForInfoBox + except: + # deleted by user - no longer needs updating + self.results[expIdx]["status"] = STATUS_user_cleared_job + return _tUpdateCount = 0 _tUpdateCountTimeout = 120 # 2 mins _lastUpdate_jobTXT = "" while True: time.sleep(1) # update every sec and exit on break - if self.JobsMgmt.hasSubmittedJob(expIdx): + if _infoBox == None: + break # deleted by user - no longer needs updating + if not _infoBox.visible: + break + if self.JobsMgmt.hasSubmittedJob(expIdx): if params["status"] in [STATUS_finished_job]: break elif params["status"] in [STATUS_errored_job]: @@ -1606,9 +1668,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): if rowIdx < 0: break else: - self.workerThread = RowDeletionWorkerThread(self.formLayout, expIdx) - self.workerThread.removeRowSignal.connect(self.removeRow) - self.workerThread.start() + ROW_POP_QUEUE.append(expIdx) elif JOB_COMPLETION_STR in jobTXT: self.results[expIdx]["status"] = STATUS_finished_job _infoBox.value = jobTXT @@ -1616,10 +1676,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): # we cant delete the row directly from this thread # we will use the exp_id to identify and delete the row # using pyqtSignal - self.workerThread = RowDeletionWorkerThread(self.formLayout, expIdx) - self.workerThread.removeRowSignal.connect(self.removeRow) - self.workerThread.start() - time.sleep(2) + ROW_POP_QUEUE.append(expIdx) # break - based on status elif JOB_TRIGGERED_EXC in jobTXT: self.results[expIdx]["status"] = STATUS_errored_job @@ -1627,7 +1684,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobTXT +"\n\n"+ jobERR break elif JOB_RUNNING_STR in jobTXT: - self.results[expIdx]["status"] = STATUS_running_job + self.results[expIdx]["status"] = STATUS_running_job _infoBox.value = jobTXT _tUpdateCount += 1 if _tUpdateCount > 60: @@ -1727,15 +1784,17 @@ def runInSubProcess(self, params): self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() +ROW_POP_QUEUE = [] # Emits a signal to QFormLayout on the main thread class RowDeletionWorkerThread(QThread): removeRowSignal = pyqtSignal(int, str) - def __init__(self, formLayout, strID): + def __init__(self, formLayout): super().__init__() self.formLayout = formLayout - self.deleteRow = -1 - self.stringID = strID + + def setNewInstances(self, formLayout): + self.formLayout:QFormLayout = formLayout # we might deal with race conditions with a shrinking table # find out widget and return its index @@ -1743,18 +1802,25 @@ def findWidgetRowInLayout(self, strID): layout: QFormLayout = self.formLayout for idx in range(0, layout.rowCount()): widgetItem = layout.itemAt(idx) - name_widget = widgetItem.widget() - toolTip_string = str(name_widget.toolTip) - if strID in toolTip_string: - name_widget.setParent(None) - return idx + if widgetItem is not None: + name_widget = widgetItem.widget() + toolTip_string = str(name_widget.toolTip) + if strID in toolTip_string: + name_widget.setParent(None) + return idx return -1 def run(self): - # Emit the signal to remove the row - self.deleteRow = self.findWidgetRowInLayout(self.stringID) - if self.deleteRow > -1: - self.removeRowSignal.emit(int(self.deleteRow), str(self.stringID)) + while True: + if len(ROW_POP_QUEUE) > 0: + stringID = ROW_POP_QUEUE.pop(0) + # Emit the signal to remove the row + deleteRow = self.findWidgetRowInLayout(stringID) + if deleteRow > -1: + self.removeRowSignal.emit(int(deleteRow), str(stringID)) + time.sleep(1) + else: + time.sleep(5) # VScode debugging if __name__ == "__main__": diff --git a/recOrder/tests/widget_tests/test_pydantic_model_widget.py b/recOrder/tests/widget_tests/test_pydantic_model_widget.py index ffad88f5..7f3a9bd0 100644 --- a/recOrder/tests/widget_tests/test_pydantic_model_widget.py +++ b/recOrder/tests/widget_tests/test_pydantic_model_widget.py @@ -1,18 +1,31 @@ import sys -from PyQt6.QtWidgets import QApplication, QWidget, QVBoxLayout +from PyQt6.QtWidgets import QApplication, QWidget, QVBoxLayout, QStyle +from PyQt6.QtGui import QPixmap +from PyQt6.QtCore import QByteArray from recOrder.plugin import tab_recon +PLUGIN_NAME = "recOrder: Computational Toolkit for Label-Free Imaging" +PLUGIN_ICON = "🔬" + class MainWindow(QWidget): def __init__(self): super().__init__() - recon_tab = tab_recon.Ui_Form() + recon_tab = tab_recon.Ui_ReconTab_Form() layout = QVBoxLayout() self.setLayout(layout) layout.addWidget(recon_tab.recon_tab_widget) if __name__ == "__main__": app = QApplication(sys.argv) + app.setStyle("Fusion") # Other options: "Fusion", "Windows", "macOS", "WindowsVista" + window = MainWindow() + window.setWindowTitle(PLUGIN_ICON + " " + PLUGIN_NAME + " " + PLUGIN_ICON) + + pixmapi = getattr(QStyle.StandardPixmap, "SP_TitleBarMenuButton") + icon = app.style().standardIcon(pixmapi) + window.setWindowIcon(icon) + window.show() sys.exit(app.exec_()) \ No newline at end of file From c0d769a5e00663566305740dce5f552de0f334b7 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Fri, 20 Dec 2024 15:52:09 -0500 Subject: [PATCH 12/38] - fixes for user initiated row delete while processing another check if result row is deleted by user action while processing --- recOrder/plugin/tab_recon.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 9bc58be4..22de012e 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1644,11 +1644,16 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): _tUpdateCountTimeout = 120 # 2 mins _lastUpdate_jobTXT = "" while True: - time.sleep(1) # update every sec and exit on break - if _infoBox == None: + time.sleep(1) # update every sec and exit on break + try: + if _infoBox == None: + self.results[expIdx]["status"] = STATUS_user_cleared_job + break # deleted by user - no longer needs updating + if _infoBox.value: + pass + except: + self.results[expIdx]["status"] = STATUS_user_cleared_job break # deleted by user - no longer needs updating - if not _infoBox.visible: - break if self.JobsMgmt.hasSubmittedJob(expIdx): if params["status"] in [STATUS_finished_job]: break From a348bb8b68b1d1fc45bbbed7453fc110448f2bbd Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Wed, 25 Dec 2024 16:43:58 -0500 Subject: [PATCH 13/38] major refactor to support positions in datasets - timeout of 5 min per Job - BF button error handling when not available - GUI initiates clearing logs on first run to avoid file exist errors - a single processing run now supports the CLI spawning multiple jobs --- .../cli/apply_inverse_transfer_function.py | 12 +- recOrder/cli/jobs_mgmt.py | 191 +- recOrder/plugin/main_widget.py | 6 +- recOrder/plugin/tab_recon.py | 1727 ++++++++++++----- .../test_pydantic_model_widget.py | 2 +- 5 files changed, 1384 insertions(+), 554 deletions(-) diff --git a/recOrder/cli/apply_inverse_transfer_function.py b/recOrder/cli/apply_inverse_transfer_function.py index 2f98b246..7e658ad4 100644 --- a/recOrder/cli/apply_inverse_transfer_function.py +++ b/recOrder/cli/apply_inverse_transfer_function.py @@ -343,13 +343,14 @@ def apply_inverse_transfer_function_cli( f"{gb_ram_request} GB of memory per CPU." ) executor = submitit.AutoExecutor(folder="logs") - + executor.update_parameters( slurm_array_parallelism=np.min([50, num_jobs]), slurm_mem_per_cpu=f"{gb_ram_request}G", slurm_cpus_per_task=cpu_request, slurm_time=60, slurm_partition="cpu", + timeout_min=jobs_mgmt.JOBS_TIMEOUT # more slurm_*** resource parameters here ) @@ -372,11 +373,14 @@ def apply_inverse_transfer_function_cli( if unique_id != "": # no unique_id means no job submission info being listened to JM.startClient() - for j in jobs: + i=0 + for j in jobs: job : submitit.Job = j job_idx : str = job.job_id - JM.putJobInListClient(unique_id, job_idx) - JM.stopClient() + position = input_position_dirpaths[i] + JM.putJobInList(job, unique_id, str(job_idx), position) + i += 1 + JM.setShorterTimeout() monitor_jobs(jobs, input_position_dirpaths) diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py index b2047136..96af8630 100644 --- a/recOrder/cli/jobs_mgmt.py +++ b/recOrder/cli/jobs_mgmt.py @@ -1,35 +1,33 @@ -import os, json +import os, json, shutil import socket -from pathlib import Path -from tempfile import TemporaryDirectory import submitit +import threading, time -# Jobs query object -# Todo: Not sure where these should functions should reside - ask Talon - -# def modify_dict(shared_var_jobs, k, v, lock): -# with lock: -# for key in shared_var_jobs.keys(): -# print(key) -# tmp = shared_var_jobs[key] -# tmp["count"] += 1 -# shared_var_jobs[key] = tmp -# shared_var_jobs[k] = {"count":0, "val": v} -# print(shared_var_jobs) - -SERVER_PORT = 8089 - +SERVER_PORT = 8089 # Choose an available port +JOBS_TIMEOUT = 5 # 5 mins +SERVER_uIDsjobIDs = {} # uIDsjobIDs[uid][jid] = job class JobsManagement(): def __init__(self, *args, **kwargs): - # self.m = Manager() - # self.shared_var_jobs = self.m.dict() - # self.lock = self.m.Lock() - self.shared_var_jobs = dict() self.executor = submitit.AutoExecutor(folder="logs") self.logsPath = self.executor.folder - self.tmp_path = TemporaryDirectory() - self.tempDir = os.path.join(Path(self.tmp_path.name), "tempfiles") + self.clientsocket = None + self.uIDsjobIDs = {} # uIDsjobIDs[uid][jid] = job + + def clearLogs(self): + thread = threading.Thread(target=self.clearLogFiles, args={self.logsPath,}) + thread.start() + + def clearLogFiles(self, dirPath): + for filename in os.listdir(dirPath): + file_path = os.path.join(dirPath, filename) + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except Exception as e: + print('Failed to delete %s. Reason: %s' % (file_path, e)) def checkForJobIDFile(self, jobID, extension="out"): files = os.listdir(self.logsPath) @@ -45,6 +43,9 @@ def checkForJobIDFile(self, jobID, extension="out"): except Exception as exc: print(exc.args) return "" + + def setShorterTimeout(self): + self.clientsocket.settimeout(3) def startClient(self): try: @@ -52,49 +53,125 @@ def startClient(self): self.clientsocket.settimeout(300) self.clientsocket.connect(('localhost', SERVER_PORT)) self.clientsocket.settimeout(None) + + thread = threading.Thread(target=self.stopClient) + thread.start() except Exception as exc: print(exc.args) def stopClient(self): try: - self.clientsocket.close() + time.sleep(2) + while True: + time.sleep(1) + buf = "" + try: + buf = self.clientsocket.recv(1024) + except: + pass + if len(buf) > 0: + if b"\n" in buf: + dataList = buf.split(b"\n") + for data in dataList: + if len(data)>0: + decoded_string = data.decode() + json_str = str(decoded_string) + json_obj = json.loads(json_str) + u_idx = json_obj["uID"] + job_idx = str(json_obj["jID"]) + cmd = json_obj["command"] + if cmd == "clientRelease": + if self.hasSubmittedJob(u_idx, job_idx): + self.clientsocket.close() + break + if cmd == "cancel": + if self.hasSubmittedJob(u_idx, job_idx): + try: + job = self.uIDsjobIDs[u_idx][job_idx] + job.cancel() + except Exception as exc: + pass # possibility of throwing an exception based on diff. OS + forDeletions = [] + for uID in self.uIDsjobIDs.keys(): + for jID in self.uIDsjobIDs[uID].keys(): + job = self.uIDsjobIDs[uID][jID] + if job.done(): + forDeletions.append((uID, jID)) + for idx in range(len(forDeletions)): + del self.uIDsjobIDs[forDeletions[idx][0]][forDeletions[idx][1]] + forDeletions = [] + for uID in self.uIDsjobIDs.keys(): + if len(self.uIDsjobIDs[uID].keys()) == 0: + forDeletions.append(uID) + for idx in range(len(forDeletions)): + del self.uIDsjobIDs[forDeletions[idx]] + if len(self.uIDsjobIDs.keys()) == 0: + self.clientsocket.close() + break except Exception as exc: + self.clientsocket.close() print(exc.args) - def putJobInListClient(self, uid: str, job: str): + def checkAllExpJobsCompletion(self, uID): + if uID in self.uIDsjobIDs.keys(): + for jobEntry in self.uIDsjobIDs[uID]: + jobsBool = jobEntry["jID"] + if jobsBool == False: + return False + return True + + def putJobCompletionInList(self, jobBool, uID: str, jID: str, mode="client"): + if uID in self.uIDsjobIDs.keys(): + if jID in self.uIDsjobIDs[uID].keys(): + self.uIDsjobIDs[uID][jID] = jobBool + + def putJobInList(self, job, uID: str, jID: str, well:str, mode="client"): try: - json_obj = {uid:job} - json_str = json.dumps(json_obj) - self.clientsocket.send(bytes(json_str, 'UTF-8')) - # p1 = Process(target=modify_dict, args=(self.shared_var_jobs, uid, job, self.lock)) - # p1.start() - # p1.join() - # p2 = Process(target=increment, args=(self.shared_var_jobs, self.lock)) - # p2.start() - # p2.join() + well = str(well) + if ".zarr" in well: + wells = well.split(".zarr") + well = wells[1].replace("\\","-").replace("/","-")[1:] + if mode == "client": + if uID not in self.uIDsjobIDs.keys(): + self.uIDsjobIDs[uID] = {} + self.uIDsjobIDs[uID][jID] = job + else: + if jID not in self.uIDsjobIDs[uID].keys(): + self.uIDsjobIDs[uID][jID] = job + json_obj = {uID:{"jID": str(jID), "pos": well}} + json_str = json.dumps(json_obj)+"\n" + self.clientsocket.send(json_str.encode()) + else: + # from server side jobs object entry is a None object + # this will be later checked as completion boolean for a ExpID which might + # have several Jobs associated with it + if uID not in SERVER_uIDsjobIDs.keys(): + SERVER_uIDsjobIDs[uID] = {} + SERVER_uIDsjobIDs[uID][jID] = job + else: + if jID not in SERVER_uIDsjobIDs[uID].keys(): + SERVER_uIDsjobIDs[uID][jID] = job except Exception as exc: print(exc.args) - def hasSubmittedJob(self, uuid_str: str)->bool: - if uuid_str in self.shared_var_jobs.keys(): - return True - return False + def hasSubmittedJob(self, uID: str, mode="client")->bool: + if mode == "client": + if uID in self.uIDsjobIDs.keys(): + return True + return False + else: + if uID in SERVER_uIDsjobIDs.keys(): + return True + return False - ####### below - not being used ######### - - def getJobs(self): - return self.shared_var_jobs - - def getJob(self, uuid_str: str)->submitit.Job: - if uuid_str in self.shared_var_jobs.keys(): - return self.shared_var_jobs[uuid_str] - - def cancelJob(self, uuid_str: str): - if uuid_str in self.shared_var_jobs.keys(): - job: submitit.Job = self.shared_var_jobs[uuid_str] - job.cancel() - - def getInfoOnJob(self, uuid_str: str)->list: - if uuid_str in self.shared_var_jobs.keys(): - job: submitit.Job = self.shared_var_jobs[uuid_str] - return job.results() + def hasSubmittedJob(self, uID: str, jID: str, mode="client")->bool: + if mode == "client": + if uID in self.uIDsjobIDs.keys(): + if jID in self.uIDsjobIDs[uID].keys(): + return True + return False + else: + if uID in SERVER_uIDsjobIDs.keys(): + if jID in SERVER_uIDsjobIDs[uID].keys(): + return True + return False diff --git a/recOrder/plugin/main_widget.py b/recOrder/plugin/main_widget.py index 402d28d2..d27b52a6 100644 --- a/recOrder/plugin/main_widget.py +++ b/recOrder/plugin/main_widget.py @@ -909,7 +909,11 @@ def connect_to_mm(self): raise KeyError(msg) if not self.bf_channel_found: - self.ui.qbutton_acq_phase_from_bf.disconnect() + try: + self.ui.qbutton_acq_phase_from_bf.disconnect() + except Exception as exc: + print(exc.args) + logging.debug(exc.args) self.ui.qbutton_acq_phase_from_bf.setStyleSheet( self.disabled_button_style ) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 22de012e..d642cb62 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -7,8 +7,7 @@ from magicgui.widgets import * from PyQt6.QtCore import pyqtSignal -from iohub.ngff import Plate, open_ome_zarr -from natsort import natsorted +from iohub.ngff import open_ome_zarr from typing import List, Literal, Union, Final, Annotated from magicgui import widgets @@ -37,10 +36,14 @@ try: # Use version specific pydantic import for ModelMetaclass # prefer to pin to 1.10.19 - version = importlib.metadata.version('pydantic') + version = importlib.metadata.version("pydantic") # print("Your Pydantic library ver:{v}.".format(v=version)) if version >= "2.0.0": - print("Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format(v=version)) + print( + "Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format( + v=version + ) + ) from pydantic.main import ModelMetaclass from pydantic.main import ValidationError from pydantic.main import BaseModel @@ -49,7 +52,11 @@ from pydantic.main import ValidationError from pydantic.main import BaseModel else: - print("Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format(v=version)) + print( + "Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format( + v=version + ) + ) from pydantic.v1.main import ModelMetaclass from pydantic.v1.main import ValidationError from pydantic.v1.main import BaseModel @@ -65,21 +72,22 @@ STATUS_errored_pool = "Errored_Pool" STATUS_errored_job = "Errored_Job" STATUS_user_cleared_job = "User_Cleared_Job" +STATUS_user_cancelled_job = "User_Cancelled_Job" -MSG_SUCCESS = {'msg':'success'} +MSG_SUCCESS = {"msg": "success"} JOB_COMPLETION_STR = "Job completed successfully" JOB_RUNNING_STR = "Starting with JobEnvironment" JOB_TRIGGERED_EXC = "Submitted job triggered an exception" -_validate_alert = '⚠' -_validate_ok = '✔️' +_validate_alert = "⚠" +_validate_ok = "✔️" # For now replicate CLI processing modes - these could reside in the CLI settings file as well # for consistency OPTION_TO_MODEL_DICT = { - "birefringence": {"enabled":False, "setting":None}, - "phase": {"enabled":False, "setting":None}, - "fluorescence": {"enabled":False, "setting":None}, + "birefringence": {"enabled": False, "setting": None}, + "phase": {"enabled": False, "setting": None}, + "fluorescence": {"enabled": False, "setting": None}, } CONTAINERS_INFO = {} @@ -88,16 +96,21 @@ # napari will not stop processes and the Hide event is not reliable HAS_INSTANCE = {"val": False, "instance": None} +# Components Queue list for new Jobs spanned from single processing +NEW_WIDGETS_QUEUE = [] +NEW_WIDGETS_QUEUE_THREADS = [] +MULTI_JOBS_REFS = {} + # Main class for the Reconstruction tab # Not efficient since instantiated from GUI # Does not have access to common functions in main_widget # ToDo : From main_widget and pass self reference class Ui_ReconTab_Form(QWidget): - - def __init__(self, parent=None): + + def __init__(self, parent=None, stand_alone=False): super().__init__(parent) self._ui = parent - + self.stand_alone = stand_alone if HAS_INSTANCE["val"]: self.current_dir_path = str(Path.cwd()) self.current_save_path = HAS_INSTANCE["current_save_path"] @@ -111,50 +124,52 @@ def __init__(self, parent=None): self.input_directory = str(Path.cwd()) self.save_directory = str(Path.cwd()) self.model_directory = str(Path.cwd()) - self.yaml_model_file = str(Path.cwd()) - + self.yaml_model_file = str(Path.cwd()) + self.input_directory_dataset = None self.input_directory_datasetMeta = None # Top level parent self.recon_tab_widget = QWidget() self.recon_tab_layout = QVBoxLayout() - self.recon_tab_layout.setAlignment(Qt.AlignTop) - self.recon_tab_layout.setContentsMargins(0,0,0,0) - self.recon_tab_layout.setSpacing(0) - self.recon_tab_widget.setLayout(self.recon_tab_layout) + self.recon_tab_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.recon_tab_layout.setContentsMargins(0, 0, 0, 0) + self.recon_tab_layout.setSpacing(0) + self.recon_tab_widget.setLayout(self.recon_tab_layout) - # Top level - Data Input + # Top level - Data Input self.modes_widget2 = QWidget() self.modes_layout2 = QHBoxLayout() - self.modes_layout2.setAlignment(Qt.AlignTop) + self.modes_layout2.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) self.modes_widget2.setLayout(self.modes_layout2) self.modes_widget2.setMaximumHeight(50) - self.modes_widget2.setMinimumHeight(50) + self.modes_widget2.setMinimumHeight(50) self.reconstruction_input_data_loc = widgets.LineEdit( - name="", - value=self.input_directory + name="", value=self.input_directory ) self.reconstruction_input_data_btn = widgets.PushButton( - name="InputData", - label="Input Data" + name="InputData", label="Input Data" + ) + self.reconstruction_input_data_btn.clicked.connect( + self.browse_dir_path_input + ) + self.reconstruction_input_data_loc.changed.connect( + self.readAndSetInputPathOnValidation ) - self.reconstruction_input_data_btn.clicked.connect(self.browse_dir_path_input) - self.reconstruction_input_data_loc.changed.connect(self.readAndSetInputPathOnValidation) self.modes_layout2.addWidget(self.reconstruction_input_data_loc.native) self.modes_layout2.addWidget(self.reconstruction_input_data_btn.native) - self.recon_tab_layout.addWidget(self.modes_widget2) - + self.recon_tab_layout.addWidget(self.modes_widget2) + # Top level - Selection modes, model creation and running self.modes_widget = QWidget() self.modes_layout = QHBoxLayout() - self.modes_layout.setAlignment(Qt.AlignTop) + self.modes_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) self.modes_widget.setLayout(self.modes_layout) self.modes_widget.setMaximumHeight(50) self.modes_widget.setMinimumHeight(50) - + self.modes_selected = OPTION_TO_MODEL_DICT.copy() # Make a copy of the Reconstruction settings mode, these will be used as template @@ -164,26 +179,27 @@ def __init__(self, parent=None): # Checkboxes for the modes to select single or combination of modes for mode in self.modes_selected.keys(): self.modes_selected[mode]["Checkbox"] = widgets.Checkbox( - name=mode, - label=mode + name=mode, label=mode + ) + self.modes_layout.addWidget( + self.modes_selected[mode]["Checkbox"].native ) - self.modes_layout.addWidget(self.modes_selected[mode]["Checkbox"].native) # PushButton to create a copy of the model - UI self.reconstruction_mode_enabler = widgets.PushButton( - name="CreateModel", - label="Create Model" + name="CreateModel", label="Create Model" + ) + self.reconstruction_mode_enabler.clicked.connect( + self._create_acq_contols ) - self.reconstruction_mode_enabler.clicked.connect(self._create_acq_contols) # PushButton to validate and create the yaml file(s) based on selection self.build_button = widgets.PushButton(name="Build && Run Model") self.build_button.clicked.connect(self.build_model_and_run) - + # PushButton to clear all copies of models that are create for UI self.reconstruction_mode_clear = widgets.PushButton( - name="ClearModels", - label="Clear All Models" + name="ClearModels", label="Clear All Models" ) self.reconstruction_mode_clear.clicked.connect(self._clear_all_models) @@ -196,104 +212,123 @@ def __init__(self, parent=None): self.modes_layout.addWidget(self.reconstruction_mode_clear.native) self.recon_tab_layout.addWidget(self.modes_widget) - _load_model_loc = widgets.LineEdit( - name="", - value=self.model_directory - ) - _load_model_btn = widgets.PushButton( - name="LoadModel", - label="Load Model" - ) + _load_model_loc = widgets.LineEdit(name="", value=self.model_directory) + # _load_model_btn = widgets.PushButton( + # name="LoadModel", label="Load Model" + # ) + _load_model_btn = DropButton(text="Load Model", recon_tab=self) # Passing model location label to model location selector - _load_model_btn.clicked.connect(lambda: self.browse_dir_path_model(_load_model_loc)) + _load_model_btn.clicked.connect( + lambda: self.browse_dir_path_model(_load_model_loc) + ) _clear_results_btn = widgets.PushButton( - name="ClearResults", - label="Clear Results" + name="ClearResults", label="Clear Results" ) _clear_results_btn.clicked.connect(self.clear_results_table) - + # HBox for Loading Model _hBox_widget_model = QWidget() _hBox_layout_model = QHBoxLayout() - _hBox_layout_model.setAlignment(Qt.AlignTop) + _hBox_layout_model.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _hBox_widget_model.setLayout(_hBox_layout_model) _hBox_widget_model.setMaximumHeight(50) _hBox_widget_model.setMinimumHeight(50) _hBox_layout_model.addWidget(_load_model_loc.native) - _hBox_layout_model.addWidget(_load_model_btn.native) + _hBox_layout_model.addWidget(_load_model_btn) _hBox_layout_model.addWidget(_clear_results_btn.native) self.recon_tab_layout.addWidget(_hBox_widget_model) - # Line seperator between pydantic UI components + # Line seperator between top / middle UI components _line = QFrame() _line.setMinimumWidth(1) _line.setFixedHeight(2) _line.setFrameShape(QFrame.HLine) _line.setFrameShadow(QFrame.Sunken) _line.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) - _line.setStyleSheet("margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;") + _line.setStyleSheet( + "margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;" + ) self.recon_tab_layout.addWidget(_line) - + # Top level - Central scrollable component which will hold Editable/(vertical) Expanding UI self.recon_tab_scrollArea_settings = QScrollArea() # self.recon_tab_scrollArea_settings.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.recon_tab_scrollArea_settings.setWidgetResizable(True) - self.recon_tab_qwidget_settings = QWidget() + self.recon_tab_qwidget_settings = DropWidget(self) self.recon_tab_qwidget_settings_layout = QVBoxLayout() self.recon_tab_qwidget_settings_layout.setSpacing(10) - self.recon_tab_qwidget_settings_layout.setAlignment(Qt.AlignTop) - self.recon_tab_qwidget_settings.setLayout(self.recon_tab_qwidget_settings_layout) - self.recon_tab_scrollArea_settings.setWidget(self.recon_tab_qwidget_settings) - self.recon_tab_layout.addWidget(self.recon_tab_scrollArea_settings) - - _line2 = QFrame() - _line2.setMinimumWidth(1) - _line2.setFixedHeight(2) - _line2.setFrameShape(QFrame.HLine) - _line2.setFrameShadow(QFrame.Sunken) - _line2.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) - _line2.setStyleSheet("margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;") - self.recon_tab_layout.addWidget(_line2) + self.recon_tab_qwidget_settings_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.recon_tab_qwidget_settings.setLayout( + self.recon_tab_qwidget_settings_layout + ) + self.recon_tab_scrollArea_settings.setWidget( + self.recon_tab_qwidget_settings + ) + + # Create the splitter + splitter = QSplitter(self) + splitter.setOrientation(Qt.Orientation.Vertical) + splitter.setSizes([600, 200]) + self.recon_tab_layout.addWidget(splitter) + + # self.recon_tab_layout.addWidget(self.recon_tab_scrollArea_settings) + splitter.addWidget(self.recon_tab_scrollArea_settings) + _scrollArea = QScrollArea() - # _scrollArea.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) - _scrollArea.setWidgetResizable(True) - _scrollArea.setMaximumHeight(200) + _scrollArea.setWidgetResizable(True) _qwidget_settings = QWidget() + _qwidget_settings.setSizePolicy( + QSizePolicy.Expanding, QSizePolicy.Expanding + ) + # _qwidget_settings.setStyleSheet( + # "margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;" + # ) _qwidget_settings_layout = QVBoxLayout() - _qwidget_settings_layout.setAlignment(Qt.AlignTop) + _qwidget_settings_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _qwidget_settings.setLayout(_qwidget_settings_layout) _scrollArea.setWidget(_qwidget_settings) - self.recon_tab_layout.addWidget(_scrollArea) + splitter.addWidget(_scrollArea) + + my_splitter_handle = splitter.handle(1) + my_splitter_handle.setStyleSheet("background: 1px rgb(128,128,128);") + splitter.setStyleSheet("""QSplitter::handle:pressed {background-color: #ca5;}""") # Table for processing entries self.proc_table_QFormLayout = QFormLayout() + self.proc_table_QFormLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) self.proc_table_QFormLayout.setSpacing(0) - self.proc_table_QFormLayout.setContentsMargins(0,0,0,0) + self.proc_table_QFormLayout.setContentsMargins(0, 0, 0, 0) _proc_table_widget = QWidget() - _proc_table_widget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) + _proc_table_widget.setSizePolicy( + QSizePolicy.Expanding, QSizePolicy.Expanding + ) _proc_table_widget.setLayout(self.proc_table_QFormLayout) - _qwidget_settings_layout.addWidget(_proc_table_widget) + _qwidget_settings_layout.addWidget(_proc_table_widget) # Stores Model & Components values which cause validation failure - can be highlighted on the model field as Red self.modelHighlighterVals = {} - + # Flag to delete Process update table row on successful Job completion # self.autoDeleteRowOnCompletion = True # handle napari's close widget and avoid starting a second server if HAS_INSTANCE["val"]: - self.worker:MyWorker = HAS_INSTANCE["MyWorker"] - self.worker.setNewInstances(self.proc_table_QFormLayout, self, self._ui) + self.worker: MyWorker = HAS_INSTANCE["MyWorker"] + self.worker.setNewInstances( + self.proc_table_QFormLayout, self, self._ui + ) else: self.worker = MyWorker(self.proc_table_QFormLayout, self, self._ui) HAS_INSTANCE["val"] = True HAS_INSTANCE["MyWorker"] = self.worker app = QApplication.instance() - app.lastWindowClosed.connect(self.myCloseEvent) # this line is connection to signal close + app.lastWindowClosed.connect( + self.myCloseEvent + ) # this line is connection to signal close # our defined close event since napari doesnt do def myCloseEvent(self): @@ -306,7 +341,9 @@ def closeEvent(self, event): self.worker.stopServer() def hideEvent(self, event): - if event.type() == QEvent.Type.Hide and (self._ui is not None and self._ui.isVisible()): + if event.type() == QEvent.Type.Hide and ( + self._ui is not None and self._ui.isVisible() + ): pass def showEvent(self, event): @@ -315,7 +352,12 @@ def showEvent(self, event): def confirmDialog(self): qm = QMessageBox - ret = qm.question(self.recon_tab_widget, "Confirm", "Confirm your selection ?", qm.Yes | qm.No) + ret = qm.question( + self.recon_tab_widget, + "Confirm", + "Confirm your selection ?", + qm.Yes | qm.No, + ) if ret == qm.Yes: return True else: @@ -326,9 +368,9 @@ def confirmDialog(self): # Input data selector def browse_dir_path_input(self): result = self._open_file_dialog(self.input_directory, "dir") - if result == '': + if result == "": return - + ret, ret_msg = self.validateInputData(result) if not ret: self.messageBox(ret_msg) @@ -343,9 +385,9 @@ def browse_dir_path_input(self): def browse_dir_path_inputBG(self, elem): result = self._open_file_dialog(self.directory, "dir") - if result == '': + if result == "": return - + ret, ret_msg = self.validateInputData(result) if not ret: self.messageBox(ret_msg) @@ -354,7 +396,9 @@ def browse_dir_path_inputBG(self, elem): elem.value = result # not working - not used - def validateInputData(self, input_data_folder: str, metadata=False) -> bool: + def validateInputData( + self, input_data_folder: str, metadata=False + ) -> bool: # Sort and validate the input paths, expanding plates into lists of positions # return True, MSG_SUCCESS try: @@ -363,16 +407,21 @@ def validateInputData(self, input_data_folder: str, metadata=False) -> bool: # ToDo: Metadata reading and implementation in GUI for # channel names, time indicies, etc. if metadata: - self.input_directory_dataset = dataset + self.input_directory_dataset = dataset return True, MSG_SUCCESS - raise Exception("Dataset does not appear to be a valid ome-zarr storage") + raise Exception( + "Dataset does not appear to be a valid ome-zarr storage" + ) except Exception as exc: return False, exc.args # call back for input LineEdit path changed manually # include data validation def readAndSetInputPathOnValidation(self): - if self.reconstruction_input_data_loc.value is None or len(self.reconstruction_input_data_loc.value) == 0: + if ( + self.reconstruction_input_data_loc.value is None + or len(self.reconstruction_input_data_loc.value) == 0 + ): self.reconstruction_input_data_loc.value = self.input_directory self.messageBox("Input data path cannot be empty") return @@ -380,7 +429,7 @@ def readAndSetInputPathOnValidation(self): self.reconstruction_input_data_loc.value = self.input_directory self.messageBox("Input data path must point to a valid location") return - + result = self.reconstruction_input_data_loc.value valid, ret_msg = self.validateInputData(result) @@ -399,8 +448,13 @@ def readAndSetInputPathOnValidation(self): # Output data selector def browse_dir_path_output(self, elem): result = self._open_file_dialog(self.save_directory, "save") - if result == '': + if result == "": return + + save_path_exists = True if Path(result).exists() else False + elem.label = "Output Data" + ("" if not save_path_exists else (" "+_validate_alert)) + elem.tooltip ="" if not save_path_exists else "Output file exists" + self.directory = result self.save_directory = result elem.value = self.save_directory @@ -408,13 +462,17 @@ def browse_dir_path_output(self, elem): self.saveLastPaths() # call back for output LineEdit path changed manually - def readAndSetOutputPathOnValidation(self, elem): - if elem.value is None or len(elem.value) == 0: - elem.value = self.input_directory - return - result = elem.value - self.directory = result - self.save_directory = result + def readAndSetOutputPathOnValidation(self, elem1, elem2, save_path): + if elem1.value is None or len(elem1.value) == 0: + elem1.value = save_path + + save_path = elem1.value + + save_path_exists = True if Path(save_path).exists() else False + elem1.label = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data" + elem1.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") + elem2.text = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data" + elem2.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") self.saveLastPaths() @@ -422,23 +480,33 @@ def readAndSetOutputPathOnValidation(self, elem): # ToDo: utilize common functions # Output data selector def browse_dir_path_model(self, elem): - results = self._open_file_dialog(self.model_directory, "files") # returns list - if len(results) == 0 or results == '': + results = self._open_file_dialog( + self.model_directory, "files" + ) # returns list + if len(results) == 0 or results == "": return - + self.model_directory = str(Path(results[0]).parent.absolute()) self.directory = self.model_directory self.current_dir_path = self.model_directory + elem.value = str(Path(results[0]).absolute()) + if len(results) > 1: + elem.value = self.model_directory self.saveLastPaths() + self.openModelFiles(results) + + + def openModelFiles(self, results:List): pydantic_models = list() for result in results: - self.yaml_model_file = result - elem.value = self.yaml_model_file + self.yaml_model_file = result - with open(result, 'r') as yaml_in: - yaml_object = utils.yaml.safe_load(yaml_in) # yaml_object will be a list or a dict + with open(result, "r") as yaml_in: + yaml_object = utils.yaml.safe_load( + yaml_in + ) # yaml_object will be a list or a dict jsonString = json.dumps(self.convert(yaml_object)) json_out = json.loads(jsonString) json_dict = dict(json_out) @@ -446,7 +514,7 @@ def browse_dir_path_model(self, elem): selected_modes = list(OPTION_TO_MODEL_DICT.copy().keys()) exclude_modes = list(OPTION_TO_MODEL_DICT.copy().keys()) - for k in range(len(selected_modes)-1, -1, -1): + for k in range(len(selected_modes) - 1, -1, -1): if selected_modes[k] in json_dict.keys(): exclude_modes.pop(k) else: @@ -455,41 +523,69 @@ def browse_dir_path_model(self, elem): pruned_pydantic_class, ret_msg = self.buildModel(selected_modes) if pruned_pydantic_class is None: self.messageBox(ret_msg) - return + return - pydantic_model, ret_msg = self.get_model_from_file(self.yaml_model_file) + pydantic_model, ret_msg = self.get_model_from_file( + self.yaml_model_file + ) if pydantic_model is None: - if isinstance(ret_msg, List) and len(ret_msg)==2 and len(ret_msg[0]["loc"])==3 and ret_msg[0]["loc"][2] == "background_path": - pydantic_model = pruned_pydantic_class # if only background_path fails validation - json_dict["birefringence"]["apply_inverse"]["background_path"] = "" - self.messageBox("background_path:\nPath was invalid and will be reset") + if ( + isinstance(ret_msg, List) + and len(ret_msg) == 2 + and len(ret_msg[0]["loc"]) == 3 + and ret_msg[0]["loc"][2] == "background_path" + ): + pydantic_model = pruned_pydantic_class # if only background_path fails validation + json_dict["birefringence"]["apply_inverse"][ + "background_path" + ] = "" + self.messageBox( + "background_path:\nPath was invalid and will be reset" + ) else: self.messageBox(ret_msg) return else: # make sure "background_path" is valid - bg_loc = json_dict["birefringence"]["apply_inverse"]["background_path"] + bg_loc = json_dict["birefringence"]["apply_inverse"][ + "background_path" + ] if bg_loc != "": extension = os.path.splitext(bg_loc)[1] if len(extension) > 0: - bg_loc = Path(os.path.join(str(Path(bg_loc).parent.absolute()),"background.zarr")) + bg_loc = Path( + os.path.join( + str(Path(bg_loc).parent.absolute()), + "background.zarr", + ) + ) else: bg_loc = Path(os.path.join(bg_loc, "background.zarr")) - if not bg_loc.exists() or not self.validateInputData(str(bg_loc)): - self.messageBox("background_path:\nPwas invalid and will be reset") - json_dict["birefringence"]["apply_inverse"]["background_path"] = "" + if not bg_loc.exists() or not self.validateInputData( + str(bg_loc) + ): + self.messageBox( + "background_path:\nPwas invalid and will be reset" + ) + json_dict["birefringence"]["apply_inverse"][ + "background_path" + ] = "" else: - json_dict["birefringence"]["apply_inverse"]["background_path"] = str(bg_loc.parent.absolute()) - - pydantic_model = self._create_acq_contols2(selected_modes, exclude_modes, pydantic_model, json_dict) + json_dict["birefringence"]["apply_inverse"][ + "background_path" + ] = str(bg_loc.parent.absolute()) + + pydantic_model = self._create_acq_contols2( + selected_modes, exclude_modes, pydantic_model, json_dict + ) if pydantic_model is None: self.messageBox("Error - pydantic model returned None") return - + pydantic_models.append(pydantic_model) - + return pydantic_models - + # useful when using close widget and not napari close and we might need them again def saveLastPaths(self): HAS_INSTANCE["current_dir_path"] = self.current_dir_path @@ -498,7 +594,7 @@ def saveLastPaths(self): HAS_INSTANCE["save_directory"] = self.save_directory HAS_INSTANCE["model_directory"] = self.model_directory HAS_INSTANCE["yaml_model_file"] = self.yaml_model_file - + # clears the results table def clear_results_table(self): if self.confirmDialog(): @@ -513,10 +609,12 @@ def removeRow(self, row, expID): name_widget = widgetItem.widget() toolTip_string = str(name_widget.toolTip) if expID in toolTip_string: - self.proc_table_QFormLayout.removeRow(row) # removeRow vs takeRow for threads ? + self.proc_table_QFormLayout.removeRow( + row + ) # removeRow vs takeRow for threads ? except Exception as exc: print(exc.args) - + # marks fields on the Model that cause a validation error def modelHighlighter(self, errs): try: @@ -527,7 +625,9 @@ def modelHighlighter(self, errs): self.modelHighlighterVals[uid]["items"] = [] self.modelHighlighterVals[uid]["tooltip"] = [] if len(errs[uid]["errs"]) > 0: - self.modelHighlighterSetter(errs[uid]["errs"], container, uid) + self.modelHighlighterSetter( + errs[uid]["errs"], container, uid + ) except Exception as exc: print(exc.args) # more of a test feature - no need to show up @@ -547,40 +647,110 @@ def formatStringForErrorDisplay(self, errs): return ret_str # recursively fix the container for highlighting - def modelHighlighterSetter(self, errs, container:Container, containerID, lev=0): + def modelHighlighterSetter( + self, errs, container: Container, containerID, lev=0 + ): try: layout = container.native.layout() for i in range(layout.count()): item = layout.itemAt(i) if item.widget(): widget = layout.itemAt(i).widget() - if (not isinstance(widget._magic_widget, CheckBox) and not isinstance(widget._magic_widget, PushButton)) and not isinstance(widget._magic_widget, LineEdit) and isinstance(widget._magic_widget._inner_widget, Container) and not (widget._magic_widget._inner_widget is None): - self.modelHighlighterSetter(errs, widget._magic_widget._inner_widget, containerID, lev+1) + if ( + ( + not isinstance(widget._magic_widget, CheckBox) + and not isinstance( + widget._magic_widget, PushButton + ) + ) + and not isinstance(widget._magic_widget, LineEdit) + and isinstance( + widget._magic_widget._inner_widget, Container + ) + and not (widget._magic_widget._inner_widget is None) + ): + self.modelHighlighterSetter( + errs, + widget._magic_widget._inner_widget, + containerID, + lev + 1, + ) else: for idx in range(len(errs)): - if len(errs[idx]["loc"])-1 < lev: + if len(errs[idx]["loc"]) - 1 < lev: pass - elif isinstance(widget._magic_widget, CheckBox) or isinstance(widget._magic_widget, LineEdit) or isinstance(widget._magic_widget, PushButton): - if widget._magic_widget.label == errs[idx]["loc"][lev].replace("_", " "): + elif ( + isinstance(widget._magic_widget, CheckBox) + or isinstance(widget._magic_widget, LineEdit) + or isinstance(widget._magic_widget, PushButton) + ): + if widget._magic_widget.label == errs[idx][ + "loc" + ][lev].replace("_", " "): if widget._magic_widget.tooltip is None: widget._magic_widget.tooltip = "-\n" - self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget) - self.modelHighlighterVals[containerID]["tooltip"].append(widget._magic_widget.tooltip) - widget._magic_widget.tooltip += errs[idx]["msg"] + "\n" - widget._magic_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") - elif widget._magic_widget._label_widget.value == errs[idx]["loc"][lev].replace("_", " "): - if widget._magic_widget._label_widget.tooltip is None: - widget._magic_widget._label_widget.tooltip = "-\n" - self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget._label_widget) - self.modelHighlighterVals[containerID]["tooltip"].append(widget._magic_widget._label_widget.tooltip) - widget._magic_widget._label_widget.tooltip += errs[idx]["msg"] + "\n" - widget._magic_widget._label_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") - if widget._magic_widget._inner_widget.tooltip is None: - widget._magic_widget._inner_widget.tooltip = "-\n" - self.modelHighlighterVals[containerID]["items"].append(widget._magic_widget._inner_widget) - self.modelHighlighterVals[containerID]["tooltip"].append(widget._magic_widget._inner_widget.tooltip) - widget._magic_widget._inner_widget.tooltip += errs[idx]["msg"] + "\n" - widget._magic_widget._inner_widget.native.setStyleSheet("border:1px solid rgb(255, 255, 0); border-width: 1px;") + self.modelHighlighterVals[containerID][ + "items" + ].append(widget._magic_widget) + self.modelHighlighterVals[containerID][ + "tooltip" + ].append(widget._magic_widget.tooltip) + widget._magic_widget.tooltip += ( + errs[idx]["msg"] + "\n" + ) + widget._magic_widget.native.setStyleSheet( + "border:1px solid rgb(255, 255, 0); border-width: 1px;" + ) + elif ( + widget._magic_widget._label_widget.value + == errs[idx]["loc"][lev].replace("_", " ") + ): + if ( + widget._magic_widget._label_widget.tooltip + is None + ): + widget._magic_widget._label_widget.tooltip = ( + "-\n" + ) + self.modelHighlighterVals[containerID][ + "items" + ].append( + widget._magic_widget._label_widget + ) + self.modelHighlighterVals[containerID][ + "tooltip" + ].append( + widget._magic_widget._label_widget.tooltip + ) + widget._magic_widget._label_widget.tooltip += ( + errs[idx]["msg"] + "\n" + ) + widget._magic_widget._label_widget.native.setStyleSheet( + "border:1px solid rgb(255, 255, 0); border-width: 1px;" + ) + if ( + widget._magic_widget._inner_widget.tooltip + is None + ): + widget._magic_widget._inner_widget.tooltip = ( + "-\n" + ) + self.modelHighlighterVals[containerID][ + "items" + ].append( + widget._magic_widget._inner_widget + ) + self.modelHighlighterVals[containerID][ + "tooltip" + ].append( + widget._magic_widget._inner_widget.tooltip + ) + widget._magic_widget._inner_widget.tooltip += ( + errs[idx]["msg"] + "\n" + ) + widget._magic_widget._inner_widget.native.setStyleSheet( + "border:1px solid rgb(255, 255, 0); border-width: 1px;" + ) except Exception as exc: print(exc.args) # more of a test feature - no need to show up @@ -591,29 +761,36 @@ def modelResetHighlighterSetter(self): for containerID in self.modelHighlighterVals.keys(): items = self.modelHighlighterVals[containerID]["items"] tooltip = self.modelHighlighterVals[containerID]["tooltip"] - i=0 + i = 0 for widItem in items: # widItem.tooltip = None # let them tool tip remain - widItem.native.setStyleSheet("border:1px solid rgb(0, 0, 0); border-width: 0px;") + widItem.native.setStyleSheet( + "border:1px solid rgb(0, 0, 0); border-width: 0px;" + ) widItem.tooltip = tooltip[i] i += 1 - + except Exception as exc: print(exc.args) # more of a test feature - no need to show up - + except Exception as exc: print(exc.args) # more of a test feature - no need to show up # passes msg to napari notifications def messageBox(self, msg, type="exc"): - if len(msg) > 0: + if len(msg) > 0: try: json_object = msg json_txt = "" for err in json_object: - json_txt = json_txt + "Loc: {loc}\nMsg:{msg}\nType:{type}\n\n".format(loc=err["loc"], msg=err["msg"], type=err["type"]) + json_txt = ( + json_txt + + "Loc: {loc}\nMsg:{msg}\nType:{type}\n\n".format( + loc=err["loc"], msg=err["msg"], type=err["type"] + ) + ) json_txt = str(json_txt) # ToDo: format it better # formatted txt does not show up properly in msg-box ?? @@ -621,82 +798,197 @@ def messageBox(self, msg, type="exc"): json_txt = str(msg) # show is a message box - if type == "exc": - notifications.show_error(json_txt) + if self.stand_alone: + self.messageBoxStandAlone(json_txt) else: - notifications.show_info(json_txt) + if type == "exc": + notifications.show_error(json_txt) + else: + notifications.show_info(json_txt) + + def messageBoxStandAlone(self, msg): + q = QMessageBox(QMessageBox.Warning, "Message", str(msg)) + q.setStandardButtons(QMessageBox.StandardButton.Ok) + q.setIcon(QMessageBox.Icon.Warning) + q.exec_() + + def cancelJob(self, btn:PushButton): + if self.confirmDialog(): + btn.enabled = False + btn.text = btn.text + " (cancel called)" + + def add_widget(self, parentLayout:QVBoxLayout, expID, jID, tableEntryID="", pos=""): + + jID = str(jID) + _cancelJobBtntext = "Cancel Job {jID} ({posName})".format(jID=jID, posName=pos) + _cancelJobButton = widgets.PushButton( + name="JobID", label=_cancelJobBtntext, enabled=True, value=False + ) + _cancelJobButton.clicked.connect( + lambda: self.cancelJob(_cancelJobButton) + ) + _txtForInfoBox = "Updating {id}-{pos}: Please wait... \nJobID assigned: {jID} ".format( + id=tableEntryID, jID=jID, pos=pos + ) + _scrollAreaCollapsibleBoxDisplayWidget = ScrollableLabel( + text=_txtForInfoBox + ) + + _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() + _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + + _scrollAreaCollapsibleBoxWidgetLayout.addWidget( + _cancelJobButton.native + ) + _scrollAreaCollapsibleBoxWidgetLayout.addWidget( + _scrollAreaCollapsibleBoxDisplayWidget + ) + + _scrollAreaCollapsibleBoxWidget = QWidget() + _scrollAreaCollapsibleBoxWidget.setLayout( + _scrollAreaCollapsibleBoxWidgetLayout + ) + _scrollAreaCollapsibleBox = QScrollArea() + _scrollAreaCollapsibleBox.setWidgetResizable(True) + _scrollAreaCollapsibleBox.setMinimumHeight(600) + _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) + + _collapsibleBoxWidgetLayout = QVBoxLayout() + _collapsibleBoxWidgetLayout.setContentsMargins(0, 0, 0, 0) + _collapsibleBoxWidgetLayout.setSpacing(0) + _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) + + _collapsibleBoxWidget = CollapsibleBox( + tableEntryID + " - " + pos + ) # tableEntryID, tableEntryShortDesc - should update with processing status + _collapsibleBoxWidget.setSizePolicy( + QSizePolicy.Expanding, QSizePolicy.Fixed + ) + _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) + + parentLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + parentLayout.addWidget(_collapsibleBoxWidget) + + MULTI_JOBS_REFS[expID+jID] = {} + MULTI_JOBS_REFS[expID+jID]["cancelBtn"] = _cancelJobButton + MULTI_JOBS_REFS[expID+jID]["infobox"] = _scrollAreaCollapsibleBoxDisplayWidget + NEW_WIDGETS_QUEUE.remove(expID+jID) + + def addTableEntryJob(self, proc_params): + + tableEntryID = proc_params["tableEntryID"] + parentLayout:QVBoxLayout = proc_params["parent_layout"] + + _cancelJobButton = widgets.PushButton( + name="JobID", label="Cancel Job", value=False, enabled=False + ) + _cancelJobButton.clicked.connect( + lambda: self.cancelJob(_cancelJobButton) + ) + _txtForInfoBox = "Updating {id}: Please wait...".format( + id=tableEntryID + ) + _scrollAreaCollapsibleBoxDisplayWidget = ScrollableLabel( + text=_txtForInfoBox + ) + _scrollAreaCollapsibleBoxDisplayWidget.setFixedHeight(300) + + proc_params["table_entry_infoBox"] = ( + _scrollAreaCollapsibleBoxDisplayWidget + ) + proc_params["cancelJobButton"] = ( + _cancelJobButton + ) + parentLayout.addWidget( + _cancelJobButton.native + ) + parentLayout.addWidget( + _scrollAreaCollapsibleBoxDisplayWidget + ) + + return proc_params # adds processing entry to _qwidgetTabEntry_layout as row item # row item will be purged from table as processing finishes # there could be 3 tabs for this processing table status - # Running, Finished, Errored - def addTableEntry(self, tableEntryID, tableEntryShortDesc, tableEntryVals, proc_params): - - _txtForInfoBox = "Updating {id}: Please wait...".format(id=tableEntryID) - _scrollAreaCollapsibleBoxDisplayWidget = widgets.Label(value=_txtForInfoBox) # ToDo: Replace with tablular data and Stop button - + # Running, Finished, Errored + def addTableEntry( + self, tableEntryID, tableEntryShortDesc, proc_params + ): _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() - _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBoxDisplayWidget.native) - _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(Qt.AlignTop) + _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _scrollAreaCollapsibleBoxWidget = QWidget() - _scrollAreaCollapsibleBoxWidget.setLayout(_scrollAreaCollapsibleBoxWidgetLayout) + _scrollAreaCollapsibleBoxWidget.setLayout( + _scrollAreaCollapsibleBoxWidgetLayout + ) + _scrollAreaCollapsibleBoxWidget.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) _scrollAreaCollapsibleBox = QScrollArea() _scrollAreaCollapsibleBox.setWidgetResizable(True) - _scrollAreaCollapsibleBox.setMinimumHeight(200) _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) + _scrollAreaCollapsibleBox.setMinimumHeight(600) + _scrollAreaCollapsibleBox.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) _collapsibleBoxWidgetLayout = QVBoxLayout() - _collapsibleBoxWidgetLayout.setContentsMargins(0,0,0,0) + _collapsibleBoxWidgetLayout.setContentsMargins(0, 0, 0, 0) _collapsibleBoxWidgetLayout.setSpacing(0) _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) - _collapsibleBoxWidget = CollapsibleBox(tableEntryID) # tableEntryID, tableEntryShortDesc - should update with processing status - _collapsibleBoxWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + _collapsibleBoxWidget = CollapsibleBox(tableEntryID) + _collapsibleBoxWidget.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) - + _expandingTabEntryWidgetLayout = QVBoxLayout() + _expandingTabEntryWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _expandingTabEntryWidgetLayout.addWidget(_collapsibleBoxWidget) _expandingTabEntryWidget = QWidget() _expandingTabEntryWidget.toolTip = tableEntryShortDesc _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.layout().setContentsMargins(0,0,0,0) + _expandingTabEntryWidget.layout().setContentsMargins(0, 0, 0, 0) _expandingTabEntryWidget.layout().setSpacing(0) - _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + _expandingTabEntryWidget.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) + + proc_params["tableEntryID"] = tableEntryID + proc_params["parent_layout"] = _scrollAreaCollapsibleBoxWidgetLayout + proc_params = self.addTableEntryJob(proc_params) - _scrollAreaCollapsibleBoxDisplayWidget.changed.connect(lambda: self.removeTableRow(_expandingTabEntryWidget, _scrollAreaCollapsibleBoxDisplayWidget)) - # instead of adding, insert at 0 to keep latest entry on top # self.proc_table_QFormLayout.addRow(_expandingTabEntryWidget) self.proc_table_QFormLayout.insertRow(0, _expandingTabEntryWidget) proc_params["table_layout"] = self.proc_table_QFormLayout proc_params["table_entry"] = _expandingTabEntryWidget - proc_params["table_entry_infoBox"] = _scrollAreaCollapsibleBoxDisplayWidget - + self.worker.runInPool(proc_params) # result = self.worker.getResult(proc_params["exp_id"]) # print(result) - # Builds the model as required def buildModel(self, selected_modes): try: b = None p = None f = None - chNames = ['State0'] + chNames = ["State0"] exclude_modes = ["birefringence", "phase", "fluorescence"] if "birefringence" in selected_modes and "phase" in selected_modes: b = settings.BirefringenceSettings() p = settings.PhaseSettings() - chNames = ['State0','State1','State2','State3'] + chNames = ["State0", "State1", "State2", "State3"] exclude_modes = ["fluorescence"] elif "birefringence" in selected_modes: b = settings.BirefringenceSettings() - chNames = ['State0','State1','State2','State3'] + chNames = ["State0", "State1", "State2", "State3"] exclude_modes = ["fluorescence", "phase"] elif "phase" in selected_modes: p = settings.PhaseSettings() @@ -704,27 +996,41 @@ def buildModel(self, selected_modes): elif "fluorescence" in selected_modes: f = settings.FluorescenceSettings() exclude_modes = ["birefringence", "phase"] - + model = None try: - model = settings.ReconstructionSettings(input_channel_names=chNames, birefringence=b, phase=p, fluorescence=f) + model = settings.ReconstructionSettings( + input_channel_names=chNames, + birefringence=b, + phase=p, + fluorescence=f, + ) except ValidationError as exc: # use v1 and v2 differ for ValidationError - newer one is not caught properly return None, exc.errors() - - model = self._fix_model(model, exclude_modes, 'input_channel_names', chNames) + + model = self._fix_model( + model, exclude_modes, "input_channel_names", chNames + ) return model, "+".join(selected_modes) + ": MSG_SUCCESS" - + except Exception as exc: return None, exc.args - + # ToDo: Temporary fix to over ride the 'input_channel_names' default value # Needs revisitation def _fix_model(self, model, exclude_modes, attr_key, attr_val): try: for mode in exclude_modes: - model = settings.ReconstructionSettings.copy(model, exclude={mode}, deep=True, update={attr_key:attr_val}) - settings.ReconstructionSettings.__setattr__(model, attr_key, attr_val) + model = settings.ReconstructionSettings.copy( + model, + exclude={mode}, + deep=True, + update={attr_key: attr_val}, + ) + settings.ReconstructionSettings.__setattr__( + model, attr_key, attr_val + ) if hasattr(model, attr_key): model.__fields__[attr_key].default = attr_val model.__fields__[attr_key].field_info.default = attr_val @@ -733,8 +1039,8 @@ def _fix_model(self, model, exclude_modes, attr_key, attr_val): return model # Creates UI controls from model based on selections - def _create_acq_contols(self): - + def _create_acq_contols(self): + # Make a copy of selections and unsed for deletion selected_modes = [] exclude_modes = [] @@ -748,8 +1054,10 @@ def _create_acq_contols(self): self._create_acq_contols2(selected_modes, exclude_modes) - def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None, json_dict=None): - + def _create_acq_contols2( + self, selected_modes, exclude_modes, myLoadedModel=None, json_dict=None + ): + # initialize the top container and specify what pydantic class to map from if myLoadedModel is not None: pydantic_class = myLoadedModel @@ -765,41 +1073,63 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None # Container holding the pydantic UI components # Multiple instances/copies since more than 1 might be created - recon_pydantic_container = widgets.Container(name=_str, scrollable=False) + recon_pydantic_container = widgets.Container( + name=_str, scrollable=False + ) - self.add_pydantic_to_container(pydantic_class, recon_pydantic_container, exclude_modes, json_dict) + self.add_pydantic_to_container( + pydantic_class, recon_pydantic_container, exclude_modes, json_dict + ) # Run a validation check to see if the selected options are permitted # before we create the GUI # get the kwargs from the container/class pydantic_kwargs = {} - pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args(recon_pydantic_container, pydantic_class, pydantic_kwargs, exclude_modes) + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( + recon_pydantic_container, + pydantic_class, + pydantic_kwargs, + exclude_modes, + ) if pydantic_kwargs is None: self.messageBox(ret_msg) return # For list element, this needs to be cleaned and parsed back as an array - input_channel_names, ret_msg = self.clean_string_for_list("input_channel_names", pydantic_kwargs["input_channel_names"]) + input_channel_names, ret_msg = self.clean_string_for_list( + "input_channel_names", pydantic_kwargs["input_channel_names"] + ) if input_channel_names is None: self.messageBox(ret_msg) return pydantic_kwargs["input_channel_names"] = input_channel_names - time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) + time_indices, ret_msg = self.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) if time_indices is None: self.messageBox(ret_msg) return pydantic_kwargs["time_indices"] = time_indices if "birefringence" in pydantic_kwargs.keys(): - background_path, ret_msg = self.clean_path_string_when_empty("background_path", pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"]) + background_path, ret_msg = self.clean_path_string_when_empty( + "background_path", + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ], + ) if background_path is None: self.messageBox(ret_msg) return - pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"] = background_path - + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ] = background_path + # validate and return errors if None - pydantic_model, ret_msg = self.validate_pydantic_model(pydantic_class, pydantic_kwargs) + pydantic_model, ret_msg = self.validate_pydantic_model( + pydantic_class, pydantic_kwargs + ) if pydantic_model is None: self.messageBox(ret_msg) return @@ -810,7 +1140,7 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None if json_txt is None: self.messageBox(ret_msg) return - + # Line seperator between pydantic UI components _line = QFrame() _line.setMinimumWidth(1) @@ -818,97 +1148,144 @@ def _create_acq_contols2(self, selected_modes, exclude_modes, myLoadedModel=None _line.setFrameShape(QFrame.HLine) _line.setFrameShadow(QFrame.Sunken) _line.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) - _line.setStyleSheet("margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;") + _line.setStyleSheet( + "border:1px solid rgb(128,128,128); border-width: 1px;" + ) # PushButton to delete a UI container # Use case when a wrong selection of input modes get selected eg Bire+Fl # Preferably this root level validation should occur before values arevalidated # in order to display and avoid this to occur - _del_button = widgets.PushButton(name="Delete this Model") - + _del_button = widgets.PushButton(name="Delete this Model") + + c_mode = "-and-".join(selected_modes) + c_mode_short = "".join(item[:3].capitalize() for item in selected_modes) + if c_mode in CONTAINERS_INFO.keys(): + CONTAINERS_INFO[c_mode] += 1 + else: + CONTAINERS_INFO[c_mode] = 1 + num_str = "{:02d}".format(CONTAINERS_INFO[c_mode]) + c_mode_str = f"{c_mode} - {num_str}" + # Output Data location # These could be multiple based on user selection for each model # Inherits from Input by default at creation time + name_without_ext = os.path.splitext(self.input_directory)[0] + save_path = os.path.join(Path(self.input_directory).parent.absolute(), (name_without_ext + ("_recon"+c_mode_short+num_str) + ".zarr")) + save_path_exists = True if Path(save_path).exists() else False _output_data_loc = widgets.LineEdit( - name="", - value=self.input_directory + name="", value=save_path, tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") ) _output_data_btn = widgets.PushButton( - name="OutputData", - label="Output Data" + name= "OutputData", text= ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data", tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") ) - + # Passing location label to output location selector - _output_data_btn.clicked.connect(lambda: self.browse_dir_path_output(_output_data_loc)) - _output_data_loc.changed.connect(lambda: self.readAndSetOutputPathOnValidation(_output_data_loc)) + _output_data_btn.clicked.connect( + lambda: self.browse_dir_path_output(_output_data_loc) + ) + _output_data_loc.changed.connect( + lambda: self.readAndSetOutputPathOnValidation(_output_data_loc, _output_data_btn, save_path) + ) # Passing all UI components that would be deleted _expandingTabEntryWidget = QWidget() - _del_button.clicked.connect(lambda: self._delete_model(_expandingTabEntryWidget, recon_pydantic_container.native, _output_data_loc.native, _output_data_btn.native, _del_button.native, _line, _idx, _str)) - - c_mode = "-and-".join(selected_modes) - if c_mode in CONTAINERS_INFO.keys(): - CONTAINERS_INFO[c_mode] += 1 - else: - CONTAINERS_INFO[c_mode] = 1 - num_str = "{:02d}".format(CONTAINERS_INFO[c_mode]) - c_mode_str = f"{c_mode} - {num_str}" + _del_button.clicked.connect( + lambda: self._delete_model( + _expandingTabEntryWidget, + recon_pydantic_container.native, + _output_data_loc.native, + _output_data_btn.native, + _del_button.native, + _line, + _idx, + _str, + ) + ) # HBox for Output Data _hBox_widget = QWidget() _hBox_layout = QHBoxLayout() - _hBox_layout.setAlignment(Qt.AlignTop) + _hBox_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _hBox_widget.setLayout(_hBox_layout) _hBox_layout.addWidget(_output_data_loc.native) _hBox_layout.addWidget(_output_data_btn.native) # Add this container to the main scrollable widget - _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() - _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(Qt.AlignTop) + _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() + _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _scrollAreaCollapsibleBoxWidget = QWidget() - _scrollAreaCollapsibleBoxWidget.setLayout(_scrollAreaCollapsibleBoxWidgetLayout) + _scrollAreaCollapsibleBoxWidget.setLayout( + _scrollAreaCollapsibleBoxWidgetLayout + ) _scrollAreaCollapsibleBox = QScrollArea() _scrollAreaCollapsibleBox.setWidgetResizable(True) _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) - _scrollAreaCollapsibleBox.setMinimumHeight(500) - + _collapsibleBoxWidgetLayout = QVBoxLayout() - _collapsibleBoxWidgetLayout.setContentsMargins(0,0,0,0) + _collapsibleBoxWidgetLayout.setContentsMargins(0, 0, 0, 0) _collapsibleBoxWidgetLayout.setSpacing(0) _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) - _collapsibleBoxWidget = CollapsibleBox(c_mode_str) # tableEntryID, tableEntryShortDesc - should update with processing status - _collapsibleBoxWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) - _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) + _collapsibleBoxWidget = CollapsibleBox( + c_mode_str + ) # tableEntryID, tableEntryShortDesc - should update with processing status + _expandingTabEntryWidgetLayout = QVBoxLayout() _expandingTabEntryWidgetLayout.addWidget(_collapsibleBoxWidget) - + _expandingTabEntryWidget.toolTip = c_mode_str _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.layout().setContentsMargins(0,0,0,0) + _expandingTabEntryWidget.layout().setContentsMargins(0, 0, 0, 0) _expandingTabEntryWidget.layout().setSpacing(0) - _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) - _expandingTabEntryWidget.layout().setAlignment(Qt.AlignTop) + _expandingTabEntryWidget.setSizePolicy( + QSizePolicy.Expanding, QSizePolicy.Fixed + ) + _expandingTabEntryWidget.layout().setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - _scrollAreaCollapsibleBoxWidgetLayout.addWidget(recon_pydantic_container.native) + _scrollAreaCollapsibleBoxWidgetLayout.addWidget( + recon_pydantic_container.native + ) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_del_button.native) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_line) - self.recon_tab_qwidget_settings_layout.addWidget(_expandingTabEntryWidget) + _scrollAreaCollapsibleBox.setMinimumHeight(_scrollAreaCollapsibleBoxWidgetLayout.sizeHint().height()) + _collapsibleBoxWidget.setSizePolicy( + QSizePolicy.Expanding, QSizePolicy.Fixed + ) + _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) + + self.recon_tab_qwidget_settings_layout.addWidget( + _expandingTabEntryWidget + ) # Store a copy of the pydantic container along with all its associated components and properties # We dont needs a copy of the class but storing for now # This will be used for making deletion edits and looping to create our final run output # uuid - used for identiying in editable list - self.pydantic_classes.append({'uuid':_str, 'c_mode_str':c_mode_str, 'collapsibleBoxWidget':_collapsibleBoxWidget, 'class':pydantic_class, 'input':self.reconstruction_input_data_loc, 'output':_output_data_loc, 'container':recon_pydantic_container, 'selected_modes':selected_modes.copy(), 'exclude_modes':exclude_modes.copy()}) + self.pydantic_classes.append( + { + "uuid": _str, + "c_mode_str": c_mode_str, + "collapsibleBoxWidget": _collapsibleBoxWidget, + "class": pydantic_class, + "input": self.reconstruction_input_data_loc, + "output": _output_data_loc, + "container": recon_pydantic_container, + "selected_modes": selected_modes.copy(), + "exclude_modes": exclude_modes.copy(), + } + ) self.index += 1 if self.index > 1: - self.build_button.text = "Build && Run {n} Models".format(n=self.index) + self.build_button.text = "Build && Run {n} Models".format( + n=self.index + ) else: self.build_button.text = "Build && Run Model" @@ -919,7 +1296,7 @@ def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, index, _str): if not self.confirmDialog(): return False - + if wid5 is not None: wid5.setParent(None) if wid4 is not None: @@ -932,33 +1309,38 @@ def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, index, _str): wid1.setParent(None) if wid0 is not None: wid0.setParent(None) - + # Find and remove the class from our pydantic model list using uuid - i=0 + i = 0 for item in self.pydantic_classes: if item["uuid"] == _str: self.pydantic_classes.pop(i) break + i += 1 self.index = len(self.pydantic_classes) if self.index > 1: - self.build_button.text = "Build && Run {n} Models".format(n=self.index) + self.build_button.text = "Build && Run {n} Models".format( + n=self.index + ) else: self.build_button.text = "Build && Run Model" # Clear all the generated pydantic models and clears the pydantic model list def _clear_all_models(self): if self.confirmDialog(): - index = self.recon_tab_qwidget_settings_layout.count()-1 - while(index >= 0): - myWidget = self.recon_tab_qwidget_settings_layout.itemAt(index).widget() + index = self.recon_tab_qwidget_settings_layout.count() - 1 + while index >= 0: + myWidget = self.recon_tab_qwidget_settings_layout.itemAt( + index + ).widget() if myWidget is not None: myWidget.setParent(None) - index -=1 + index -= 1 self.pydantic_classes.clear() CONTAINERS_INFO.clear() self.index = 0 self.build_button.text = "Build && Run Model" - + # Displays the json output from the pydantic model UI selections by user # Loops through all our stored pydantic classes def build_model_and_run(self): @@ -967,23 +1349,27 @@ def build_model_and_run(self): # first pass for validating # second pass for creating yaml and processing - self.modelResetHighlighterSetter() # reset the container elements that might be highlighted for errors + if len(self.pydantic_classes) == 0: + self.messageBox("Please create a processing model first !") + return + + self.modelResetHighlighterSetter() # reset the container elements that might be highlighted for errors _collectAllErrors = {} _collectAllErrorsBool = True for item in self.pydantic_classes: - cls = item['class'] - cls_container = item['container'] - selected_modes = item['selected_modes'] - exclude_modes = item['exclude_modes'] - uuid_str = item['uuid'] - _collapsibleBoxWidget = item['collapsibleBoxWidget'] - c_mode_str = item['c_mode_str'] + cls = item["class"] + cls_container = item["container"] + selected_modes = item["selected_modes"] + exclude_modes = item["exclude_modes"] + uuid_str = item["uuid"] + _collapsibleBoxWidget = item["collapsibleBoxWidget"] + c_mode_str = item["c_mode_str"] _collectAllErrors[uuid_str] = {} _collectAllErrors[uuid_str]["cls"] = cls_container _collectAllErrors[uuid_str]["errs"] = [] _collectAllErrors[uuid_str]["collapsibleBox"] = c_mode_str - + # build up the arguments for the pydantic model given the current container if cls is None: self.messageBox(ret_msg) @@ -991,37 +1377,56 @@ def build_model_and_run(self): # get the kwargs from the container/class pydantic_kwargs = {} - pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args(cls_container, cls, pydantic_kwargs, exclude_modes) + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( + cls_container, cls, pydantic_kwargs, exclude_modes + ) if pydantic_kwargs is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return # For list element, this needs to be cleaned and parsed back as an array - input_channel_names, ret_msg = self.clean_string_for_list("input_channel_names", pydantic_kwargs["input_channel_names"]) + input_channel_names, ret_msg = self.clean_string_for_list( + "input_channel_names", pydantic_kwargs["input_channel_names"] + ) if input_channel_names is None and not _collectAllErrorsBool: self.messageBox(ret_msg) - return + return pydantic_kwargs["input_channel_names"] = input_channel_names - time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) + time_indices, ret_msg = self.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) if time_indices is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return pydantic_kwargs["time_indices"] = time_indices if "birefringence" in pydantic_kwargs.keys(): - background_path, ret_msg = self.clean_path_string_when_empty("background_path", pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"]) + background_path, ret_msg = self.clean_path_string_when_empty( + "background_path", + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ], + ) if background_path is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return - pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"] = background_path + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ] = background_path # validate and return errors if None - pydantic_model, ret_msg = self.validate_pydantic_model(cls, pydantic_kwargs) + pydantic_model, ret_msg = self.validate_pydantic_model( + cls, pydantic_kwargs + ) if ret_msg == MSG_SUCCESS: - _collapsibleBoxWidget.setNewName(f"{c_mode_str} {_validate_ok}") + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_ok}" + ) else: - _collapsibleBoxWidget.setNewName(f"{c_mode_str} {_validate_alert}") + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_alert}" + ) _collectAllErrors[uuid_str]["errs"] = ret_msg if pydantic_model is None and not _collectAllErrorsBool: self.messageBox(ret_msg) @@ -1033,7 +1438,7 @@ def build_model_and_run(self): if json_txt is None and not _collectAllErrorsBool: self.messageBox(ret_msg) return - + # check if we collected any validation errors before continuing for uu_key in _collectAllErrors.keys(): if len(_collectAllErrors[uu_key]["errs"]) > 0: @@ -1041,21 +1446,21 @@ def build_model_and_run(self): fmt_str = self.formatStringForErrorDisplay(_collectAllErrors) self.messageBox(fmt_str) return - + # generate a time-stamp for our yaml files to avoid overwriting # files generated at the same time will have an index suffix now = datetime.datetime.now() ms = now.strftime("%f")[:3] - unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_")+ms + unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms i = 0 for item in self.pydantic_classes: i += 1 - cls = item['class'] - cls_container = item['container'] - selected_modes = item['selected_modes'] - exclude_modes = item['exclude_modes'] - c_mode_str = item['c_mode_str'] + cls = item["class"] + cls_container = item["container"] + selected_modes = item["selected_modes"] + exclude_modes = item["exclude_modes"] + c_mode_str = item["c_mode_str"] # gather input/out locations input_dir = f"{item['input'].value}" @@ -1067,38 +1472,55 @@ def build_model_and_run(self): return pydantic_kwargs = {} - pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args(cls_container, cls, pydantic_kwargs, exclude_modes) + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( + cls_container, cls, pydantic_kwargs, exclude_modes + ) if pydantic_kwargs is None: self.messageBox(ret_msg) return - input_channel_names, ret_msg = self.clean_string_for_list("input_channel_names", pydantic_kwargs["input_channel_names"]) + input_channel_names, ret_msg = self.clean_string_for_list( + "input_channel_names", pydantic_kwargs["input_channel_names"] + ) if input_channel_names is None: self.messageBox(ret_msg) return pydantic_kwargs["input_channel_names"] = input_channel_names - time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) + time_indices, ret_msg = self.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) if time_indices is None: self.messageBox(ret_msg) return pydantic_kwargs["time_indices"] = time_indices - time_indices, ret_msg = self.clean_string_int_for_list("time_indices", pydantic_kwargs["time_indices"]) + time_indices, ret_msg = self.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) if time_indices is None: self.messageBox(ret_msg) return pydantic_kwargs["time_indices"] = time_indices if "birefringence" in pydantic_kwargs.keys(): - background_path, ret_msg = self.clean_path_string_when_empty("background_path", pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"]) + background_path, ret_msg = self.clean_path_string_when_empty( + "background_path", + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ], + ) if background_path is None: self.messageBox(ret_msg) return - pydantic_kwargs["birefringence"]["apply_inverse"]["background_path"] = background_path + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ] = background_path # validate and return errors if None - pydantic_model, ret_msg = self.validate_pydantic_model(cls, pydantic_kwargs) + pydantic_model, ret_msg = self.validate_pydantic_model( + cls, pydantic_kwargs + ) if pydantic_model is None: self.messageBox(ret_msg) return @@ -1114,7 +1536,7 @@ def build_model_and_run(self): # path is next to saved data location save_config_path = str(Path(output_dir).parent.absolute()) yml_file_name = "-and-".join(selected_modes) - yml_file = yml_file_name+"-"+unique_id+"-"+str(i)+".yml" + yml_file = yml_file_name + "-" + unique_id + "-{:02d}".format(i) + ".yml" config_path = os.path.join(save_config_path, yml_file) utils.model_to_yaml(pydantic_model, config_path) @@ -1127,9 +1549,13 @@ def build_model_and_run(self): # addl_txt = "ID:" + unique_id + "-"+ str(i) + "\nInput:" + input_dir + "\nOutput:" + output_dir # self.json_display.value = self.json_display.value + addl_txt + "\n" + json_txt+ "\n\n" - expID = "{tID}-{idx}".format(tID = unique_id, idx = i) - tableID = "{tName}: ({tID}-{idx})".format(tName = c_mode_str, tID = unique_id, idx = i) - tableDescToolTip = "{tName}: ({tID}-{idx})".format(tName = yml_file_name, tID = unique_id, idx = i) + expID = "{tID}-{idx}".format(tID=unique_id, idx=i) + tableID = "{tName}: ({tID}-{idx})".format( + tName=c_mode_str, tID=unique_id, idx=i + ) + tableDescToolTip = "{tName}: ({tID}-{idx})".format( + tName=yml_file_name, tID=unique_id, idx=i + ) proc_params = {} proc_params["exp_id"] = expID @@ -1137,10 +1563,14 @@ def build_model_and_run(self): proc_params["config_path"] = str(Path(config_path).absolute()) proc_params["input_path"] = str(Path(input_dir).absolute()) proc_params["output_path"] = str(Path(output_dir).absolute()) - proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) + proc_params["output_path_parent"] = str( + Path(output_dir).parent.absolute() + ) + + self.addTableEntry( + tableID, tableDescToolTip, proc_params + ) - self.addTableEntry(tableID, tableDescToolTip, json_txt, proc_params) - # ======= These function do not implement validation # They simply make the data from GUI translate to input types # that the model expects: for eg. GUI txt field will output only str @@ -1149,74 +1579,86 @@ def build_model_and_run(self): # util function to parse list elements displayed as string def remove_chars(self, string, chars_to_remove): for char in chars_to_remove: - string = string.replace(char, '') + string = string.replace(char, "") return string # util function to parse list elements displayed as string def clean_string_for_list(self, field, string): - chars_to_remove = ['[',']', '\'', '"', ' '] + chars_to_remove = ["[", "]", "'", '"', " "] if isinstance(string, str): string = self.remove_chars(string, chars_to_remove) if len(string) == 0: - return None, {'msg':field + ' is invalid'} - if ',' in string: - string = string.split(',') + return None, {"msg": field + " is invalid"} + if "," in string: + string = string.split(",") return string, MSG_SUCCESS if isinstance(string, str): string = [string] return string, MSG_SUCCESS return string, MSG_SUCCESS - + # util function to parse list elements displayed as string, int, int as list of strings, int range # [1,2,3], 4,5,6 , 5-95 def clean_string_int_for_list(self, field, string): - chars_to_remove = ['[',']', '\'', '"', ' '] + chars_to_remove = ["[", "]", "'", '"', " "] if Literal[string] == Literal["all"]: return string, MSG_SUCCESS if Literal[string] == Literal[""]: - return string, MSG_SUCCESS + return string, MSG_SUCCESS if isinstance(string, str): string = self.remove_chars(string, chars_to_remove) if len(string) == 0: - return None, {'msg':field + ' is invalid'} - if '-' in string: - string = string.split('-') + return None, {"msg": field + " is invalid"} + if "-" in string: + string = string.split("-") if len(string) == 2: try: x = int(string[0]) if not isinstance(x, int): raise except Exception as exc: - return None, {'msg':field + ' first range element is not an integer'} + return None, { + "msg": field + " first range element is not an integer" + } try: y = int(string[1]) if not isinstance(y, int): raise except Exception as exc: - return None, {'msg':field + ' second range element is not an integer'} + return None, { + "msg": field + + " second range element is not an integer" + } if y > x: - return list(range(x, y+1)), MSG_SUCCESS + return list(range(x, y + 1)), MSG_SUCCESS else: - return None, {'msg':field + ' second integer cannot be smaller than first'} + return None, { + "msg": field + + " second integer cannot be smaller than first" + } else: - return None, {'msg':field + ' is invalid'} - if ',' in string: - string = string.split(',') + return None, {"msg": field + " is invalid"} + if "," in string: + string = string.split(",") return string, MSG_SUCCESS return string, MSG_SUCCESS - + # util function to set path to empty - by default empty path has a "." - def clean_path_string_when_empty(self, field, string): + def clean_path_string_when_empty(self, field, string): if isinstance(string, Path) and string == Path(""): string = "" return string, MSG_SUCCESS return string, MSG_SUCCESS - + # get the pydantic_kwargs and catches any errors in doing so - def get_and_validate_pydantic_args(self, cls_container, cls, pydantic_kwargs, exclude_modes): + def get_and_validate_pydantic_args( + self, cls_container, cls, pydantic_kwargs, exclude_modes + ): try: try: - self.get_pydantic_kwargs(cls_container, cls, pydantic_kwargs, exclude_modes) + self.get_pydantic_kwargs( + cls_container, cls, pydantic_kwargs, exclude_modes + ) return pydantic_kwargs, MSG_SUCCESS except ValidationError as exc: return None, exc.errors() @@ -1225,32 +1667,36 @@ def get_and_validate_pydantic_args(self, cls_container, cls, pydantic_kwargs, ex # validate the model and return errors for user actioning def validate_pydantic_model(self, cls, pydantic_kwargs): - # instantiate the pydantic model form the kwargs we just pulled + # instantiate the pydantic model form the kwargs we just pulled try: - try : - pydantic_model = settings.ReconstructionSettings.parse_obj(pydantic_kwargs) + try: + pydantic_model = settings.ReconstructionSettings.parse_obj( + pydantic_kwargs + ) return pydantic_model, MSG_SUCCESS except ValidationError as exc: return None, exc.errors() except Exception as exc: return None, exc.args - + # test to make sure model coverts to json which should ensure compatibility with yaml export def validate_and_return_json(self, pydantic_model): - try : + try: json_format = pydantic_model.json(indent=4) return json_format, MSG_SUCCESS except Exception as exc: return None, exc.args - + # gets a copy of the model from a yaml file # will get all fields (even those that are optional and not in yaml) and default values # model needs further parsing against yaml file for fields def get_model_from_file(self, model_file_path): pydantic_model = None - try : + try: try: - pydantic_model = utils.yaml_to_model(model_file_path, settings.ReconstructionSettings) + pydantic_model = utils.yaml_to_model( + model_file_path, settings.ReconstructionSettings + ) except ValidationError as exc: return pydantic_model, exc.errors() if pydantic_model is None: @@ -1258,7 +1704,7 @@ def get_model_from_file(self, model_file_path): return pydantic_model, MSG_SUCCESS except Exception as exc: return None, exc.args - + # handles json with boolean properly and converts to lowercase string # as required def convert(self, obj): @@ -1267,7 +1713,10 @@ def convert(self, obj): if isinstance(obj, (list, tuple)): return [self.convert(item) for item in obj] if isinstance(obj, dict): - return {self.convert(key):self.convert(value) for key, value in obj.items()} + return { + self.convert(key): self.convert(value) + for key, value in obj.items() + } return obj # Main function to add pydantic model to container @@ -1279,22 +1728,30 @@ def convert(self, obj): # Displaying Union field "time_indices" as LineEdit component # excludes handles fields that are not supposed to show up from __fields__ # json_dict adds ability to provide new set of default values at time of container creation - - def add_pydantic_to_container(self, py_model:Union[BaseModel, ModelMetaclass], container: widgets.Container, excludes=[], json_dict=None): + + def add_pydantic_to_container( + self, + py_model: Union[BaseModel, ModelMetaclass], + container: widgets.Container, + excludes=[], + json_dict=None, + ): # recursively traverse a pydantic model adding widgets to a container. When a nested # pydantic model is encountered, add a new nested container - for field, field_def in py_model.__fields__.items(): + for field, field_def in py_model.__fields__.items(): if field_def is not None and field not in excludes: def_val = field_def.default - ftype = field_def.type_ - toolTip = "" + ftype = field_def.type_ + toolTip = "" try: for f_val in field_def.class_validators.keys(): toolTip = f"{toolTip}{f_val} " except Exception as e: pass - if isinstance(ftype, BaseModel) or isinstance(ftype, ModelMetaclass): + if isinstance(ftype, BaseModel) or isinstance( + ftype, ModelMetaclass + ): json_val = None if json_dict is not None: json_val = json_dict[field] @@ -1302,79 +1759,137 @@ def add_pydantic_to_container(self, py_model:Union[BaseModel, ModelMetaclass], c new_widget_cls = widgets.Container new_widget = new_widget_cls(name=field_def.name) new_widget.tooltip = toolTip - self.add_pydantic_to_container(ftype, new_widget, excludes, json_val) - #ToDo: Implement Union check, tried: + self.add_pydantic_to_container( + ftype, new_widget, excludes, json_val + ) + # ToDo: Implement Union check, tried: # pydantic.typing.is_union(ftype) # isinstance(ftype, types.UnionType) # https://stackoverflow.com/questions/45957615/how-to-check-a-variable-against-union-type-during-runtime - elif isinstance(ftype, type(Union[NonNegativeInt, List, str])): - if (field == "background_path"): #field == "background_path": - new_widget_cls, ops = get_widget_class(def_val, Annotated[Path, {"mode": "d"}], dict(name=field, value=def_val)) + elif isinstance(ftype, type(Union[NonNegativeInt, List, str])): + if ( + field == "background_path" + ): # field == "background_path": + new_widget_cls, ops = get_widget_class( + def_val, + Annotated[Path, {"mode": "d"}], + dict(name=field, value=def_val), + ) new_widget = new_widget_cls(**ops) - toolTip = "Select the folder containing background.zarr" - elif (field == "time_indices"): #field == "time_indices": - new_widget_cls, ops = get_widget_class(def_val, str, dict(name=field, value=def_val)) + toolTip = ( + "Select the folder containing background.zarr" + ) + elif field == "time_indices": # field == "time_indices": + new_widget_cls, ops = get_widget_class( + def_val, str, dict(name=field, value=def_val) + ) new_widget = new_widget_cls(**ops) - else: # other Union cases - new_widget_cls, ops = get_widget_class(def_val, str, dict(name=field, value=def_val)) + else: # other Union cases + new_widget_cls, ops = get_widget_class( + def_val, str, dict(name=field, value=def_val) + ) new_widget = new_widget_cls(**ops) new_widget.tooltip = toolTip if isinstance(new_widget, widgets.EmptyWidget): - warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + warnings.warn( + message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}" + ) elif isinstance(def_val, float): # parse the field, add appropriate widget def_step_size = 0.001 if field_def.name == "regularization_strength": def_step_size = 0.00001 if def_val > -1 and def_val < 1: - new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val, step=float(def_step_size))) + new_widget_cls, ops = get_widget_class( + None, + ftype, + dict( + name=field_def.name, + value=def_val, + step=float(def_step_size), + ), + ) new_widget = new_widget_cls(**ops) new_widget.tooltip = toolTip else: - new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val)) + new_widget_cls, ops = get_widget_class( + None, + ftype, + dict(name=field_def.name, value=def_val), + ) new_widget = new_widget_cls(**ops) new_widget.tooltip = toolTip if isinstance(new_widget, widgets.EmptyWidget): - warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + warnings.warn( + message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}" + ) else: # parse the field, add appropriate widget - new_widget_cls, ops = get_widget_class(None, ftype, dict(name=field_def.name, value=def_val)) - new_widget = new_widget_cls(**ops) + new_widget_cls, ops = get_widget_class( + None, ftype, dict(name=field_def.name, value=def_val) + ) + new_widget = new_widget_cls(**ops) if isinstance(new_widget, widgets.EmptyWidget): - warnings.warn(message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}") + warnings.warn( + message=f"magicgui could not identify a widget for {py_model}.{field}, which has type {ftype}" + ) else: - new_widget.tooltip = toolTip - if json_dict is not None and (not isinstance(new_widget, widgets.Container) or (isinstance(new_widget, widgets.FileEdit))): + new_widget.tooltip = toolTip + if json_dict is not None and ( + not isinstance(new_widget, widgets.Container) + or (isinstance(new_widget, widgets.FileEdit)) + ): if field in json_dict.keys(): if isinstance(new_widget, widgets.CheckBox): - new_widget.value = True if json_dict[field]=="true" else False + new_widget.value = ( + True if json_dict[field] == "true" else False + ) elif isinstance(new_widget, widgets.FileEdit): if len(json_dict[field]) > 0: extension = os.path.splitext(json_dict[field])[1] if len(extension) > 0: - new_widget.value = Path(json_dict[field]).parent.absolute() # CLI accepts BG folder not .zarr + new_widget.value = Path( + json_dict[field] + ).parent.absolute() # CLI accepts BG folder not .zarr else: new_widget.value = Path(json_dict[field]) else: new_widget.value = json_dict[field] container.append(new_widget) - + # refer - add_pydantic_to_container() for comments - def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pydantic_kwargs: dict, excludes=[], json_dict=None): - # given a container that was instantiated from a pydantic model, get the arguments - # needed to instantiate that pydantic model from the container. + def get_pydantic_kwargs( + self, + container: widgets.Container, + pydantic_model, + pydantic_kwargs: dict, + excludes=[], + json_dict=None, + ): + # given a container that was instantiated from a pydantic model, get the arguments + # needed to instantiate that pydantic model from the container. # traverse model fields, pull out values from container for field, field_def in pydantic_model.__fields__.items(): - if field_def is not None and field not in excludes: + if field_def is not None and field not in excludes: ftype = field_def.type_ - if isinstance(ftype, BaseModel) or isinstance(ftype, ModelMetaclass): + if isinstance(ftype, BaseModel) or isinstance( + ftype, ModelMetaclass + ): # go deeper - pydantic_kwargs[field] = {} # new dictionary for the new nest level + pydantic_kwargs[field] = ( + {} + ) # new dictionary for the new nest level # any pydantic class will be a container, so pull that out to pass # to the recursive call sub_container = getattr(container, field_def.name) - self.get_pydantic_kwargs(sub_container, ftype, pydantic_kwargs[field], excludes, json_dict) + self.get_pydantic_kwargs( + sub_container, + ftype, + pydantic_kwargs[field], + excludes, + json_dict, + ) else: # not a pydantic class, just pull the field value from the container if hasattr(container, field_def.name): @@ -1385,7 +1900,9 @@ def get_pydantic_kwargs(self, container: widgets.Container, pydantic_model, pyda # file open/select dialog def _open_file_dialog(self, default_path, type): if type == "dir": - return self._open_dialog("select a directory", str(default_path), type) + return self._open_dialog( + "select a directory", str(default_path), type + ) elif type == "file": return self._open_dialog("select a file", str(default_path), type) elif type == "files": @@ -1393,7 +1910,9 @@ def _open_file_dialog(self, default_path, type): elif type == "save": return self._open_dialog("save a file", str(default_path), type) else: - return self._open_dialog("select a directory", str(default_path), type) + return self._open_dialog( + "select a directory", str(default_path), type + ) def _open_dialog(self, title, ref, type): """ @@ -1431,7 +1950,8 @@ def _open_dialog(self, title, ref, type): raise ValueError("Did not understand file dialogue type") return path - + + class CollapsibleBox(QWidget): def __init__(self, title="", parent=None): super(CollapsibleBox, self).__init__(parent) @@ -1441,18 +1961,18 @@ def __init__(self, title="", parent=None): ) self.toggle_button.setStyleSheet("QToolButton { border: none; }") self.toggle_button.setToolButtonStyle( - QtCore.Qt.ToolButtonTextBesideIcon + QtCore.Qt.ToolButtonStyle.ToolButtonTextBesideIcon ) - self.toggle_button.setArrowType(QtCore.Qt.RightArrow) + self.toggle_button.setArrowType(QtCore.Qt.ArrowType.RightArrow) self.toggle_button.pressed.connect(self.on_pressed) self.toggle_animation = QtCore.QParallelAnimationGroup(self) self.content_area = QScrollArea(maximumHeight=0, minimumHeight=0) self.content_area.setSizePolicy( - QSizePolicy.Expanding, QSizePolicy.Fixed + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed ) - self.content_area.setFrameShape(QFrame.NoFrame) + self.content_area.setFrameShape(QFrame.Shape.NoFrame) lay = QVBoxLayout(self) lay.setSpacing(0) @@ -1473,16 +1993,16 @@ def __init__(self, title="", parent=None): def setNewName(self, name): self.toggle_button.setText(name) - # @QtCore.pyqtSlot() + # @QtCore.pyqtSlot() def on_pressed(self): checked = self.toggle_button.isChecked() self.toggle_button.setArrowType( - QtCore.Qt.DownArrow if not checked else QtCore.Qt.RightArrow + QtCore.Qt.ArrowType.DownArrow if not checked else QtCore.Qt.ArrowType.RightArrow ) self.toggle_animation.setDirection( - QtCore.QAbstractAnimation.Forward + QtCore.QAbstractAnimation.Direction.Forward if not checked - else QtCore.QAbstractAnimation.Backward + else QtCore.QAbstractAnimation.Direction.Backward ) self.toggle_animation.start() @@ -1507,37 +2027,43 @@ def setContentLayout(self, layout): content_animation.setStartValue(0) content_animation.setEndValue(content_height) + import socket, threading -class MyWorker(): - - def __init__(self, formLayout, tab_recon:Ui_ReconTab_Form, parentForm): - super().__init__() - self.formLayout:QFormLayout = formLayout - self.tab_recon:Ui_ReconTab_Form = tab_recon - self.ui:QWidget = parentForm + + +class MyWorker: + + def __init__(self, formLayout, tab_recon: Ui_ReconTab_Form, parentForm): + super().__init__() + self.formLayout: QFormLayout = formLayout + self.tab_recon: Ui_ReconTab_Form = tab_recon + self.ui: QWidget = parentForm self.max_cores = os.cpu_count() # In the case of CLI, we just need to submit requests in a non-blocking way - self.threadPool = int(self.max_cores/2) + self.threadPool = int(self.max_cores / 2) self.results = {} - self.pool = None + self.pool = None # https://click.palletsprojects.com/en/stable/testing/ # self.runner = CliRunner() self.startPool() # jobs_mgmt.shared_var_jobs = self.JobsManager.shared_var_jobs self.JobsMgmt = jobs_mgmt.JobsManagement() + self.JobsMgmt.clearLogs() self.useServer = True self.serverRunning = True - self.serverSocket = None + self.server_socket = None thread = threading.Thread(target=self.startServer) thread.start() self.workerThreadRowDeletion = RowDeletionWorkerThread(self.formLayout) - self.workerThreadRowDeletion.removeRowSignal.connect(self.tab_recon.removeRow) + self.workerThreadRowDeletion.removeRowSignal.connect( + self.tab_recon.removeRow + ) self.workerThreadRowDeletion.start() def setNewInstances(self, formLayout, tab_recon, parentForm): - self.formLayout:QFormLayout = formLayout - self.tab_recon:Ui_ReconTab_Form = tab_recon - self.ui:QWidget = parentForm + self.formLayout: QFormLayout = formLayout + self.tab_recon: Ui_ReconTab_Form = tab_recon + self.ui: QWidget = parentForm self.workerThreadRowDeletion.setNewInstances(formLayout) def findWidgetRowInLayout(self, strID): @@ -1556,46 +2082,38 @@ def startServer(self): if not self.useServer: return - self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.serverSocket.bind(('localhost', jobs_mgmt.SERVER_PORT)) - self.serverSocket.listen(50) # become a server socket, maximum 50 connections + self.server_socket = socket.socket( + socket.AF_INET, socket.SOCK_STREAM + ) + self.server_socket.bind(("localhost", jobs_mgmt.SERVER_PORT)) + self.server_socket.listen( + 50 + ) # become a server socket, maximum 50 connections while self.serverRunning: - connection, address = self.serverSocket.accept() + client_socket, address = self.server_socket.accept() if self.ui is not None and not self.ui.isVisible(): break - try: - buf = connection.recv(64) - if len(buf) > 0: - decoded_string = buf.decode("utf-8") - json_str = str(decoded_string) - json_obj = json.loads(json_str) - uID = "" - jobID = "" - for k in json_obj: - self.JobsMgmt.shared_var_jobs[k] = json_obj[k] - uID = k - jobID = json_obj[k] - - # dont block the server thread - thread = threading.Thread(target=self.tableUpdateAndCleaupThread, args=(uID, jobID)) - thread.start() + try: + # dont block the server thread + thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=("", "", "", client_socket),) + thread.start() except Exception as exc: print(exc.args) time.sleep(1) - - self.serverSocket.close() + + self.server_socket.close() except Exception as exc: if not self.serverRunning: self.serverRunning = True - return # ignore - will cause an exception on napari close but that is fine and does the job + return # ignore - will cause an exception on napari close but that is fine and does the job print(exc.args) def stopServer(self): try: - if self.serverSocket is not None: + if self.server_socket is not None: self.serverRunning = False - self.serverSocket.close() + self.server_socket.close() except Exception as exc: print(exc.args) @@ -1611,14 +2129,38 @@ def startPool(self): self.pool = ThreadPoolExecutor(max_workers=self.threadPool) def shutDownPool(self): - self.pool.shutdown(wait=False) + self.pool.shutdown(wait=True) # the table update thread can be called from multiple points/threads # on errors - table row item is updated but there is no row deletion # on successful processing - the row item is expected to be deleted # row is being deleted from a seperate thread for which we need to connect using signal - def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): + def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_socket=None): # finished will be updated by the job - submitit status + jobIdx = str(jobIdx) + if client_socket is not None and expIdx=="" and jobIdx=="": + try: + buf = client_socket.recv(1024) + if len(buf) > 0: + if b"\n" in buf: + dataList = buf.split(b"\n") + else: + dataList = [buf] + for data in dataList: + if len(data)>0: + decoded_string = data.decode() + json_str = str(decoded_string) + json_obj = json.loads(json_str) + for k in json_obj: + expIdx = k + jobIdx = json_obj[k]["jID"] + wellName = json_obj[k]["pos"] + self.JobsMgmt.putJobInList(None, expIdx, str(jobIdx), wellName, mode="server") + thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=(expIdx, jobIdx, wellName, client_socket)) + thread.start() + return + except: + pass # ToDo: Another approach to this could be to implement a status thread on the client side # Since the client is already running till the job is completed, the client could ping status @@ -1631,165 +2173,311 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx=""): # this request came from server so we can wait for the Job to finish and update progress # some wait logic needs to be added otherwise for unknown errors this thread will persist # perhaps set a time out limit and then update the status window and then exit - params = self.results[expIdx] - _infoBox:Label = params["table_entry_infoBox"] - _txtForInfoBox = "Updating {id}: Please wait... \nJobID assigned: {jID} ".format(id=params["desc"], jID=jobIdx) + params = self.results[expIdx]["JobUNK"].copy() + + if jobIdx not in self.results[expIdx].keys() and len(self.results[expIdx].keys()) == 1: + # this is the first job + params["primary"] = True + self.results[expIdx][jobIdx] = params + elif jobIdx not in self.results[expIdx].keys() and len(self.results[expIdx].keys()) > 1: + # this is a new job + # we need to create cancel and job status windows and add to parent container + params["primary"] = False + NEW_WIDGETS_QUEUE.append(expIdx+jobIdx) + parentLayout:QVBoxLayout = params["parent_layout"] + worker_thread = AddWidgetWorkerThread(parentLayout, expIdx, jobIdx, params["desc"], wellName) + worker_thread.add_widget_signal.connect(self.tab_recon.add_widget) + NEW_WIDGETS_QUEUE_THREADS.append(worker_thread) + + while (len(NEW_WIDGETS_QUEUE_THREADS) > 0): + s_worker_thread = NEW_WIDGETS_QUEUE_THREADS.pop(0) + s_worker_thread.start() + time.sleep(1) + + # wait for new components reference + while (expIdx+jobIdx in NEW_WIDGETS_QUEUE): + time.sleep(1) + + _cancelJobBtn = MULTI_JOBS_REFS[expIdx+jobIdx]["cancelBtn"] + _infoBox = MULTI_JOBS_REFS[expIdx+jobIdx]["infobox"] + params["table_entry_infoBox"] = _infoBox + params["cancelJobButton"] = _cancelJobBtn + + self.results[expIdx][jobIdx] = params + + _infoBox: ScrollableLabel = params["table_entry_infoBox"] + _cancelJobBtn: PushButton = params["cancelJobButton"] + + _txtForInfoBox = "Updating {id}-{pos}: Please wait... \nJobID assigned: {jID} ".format( + id=params["desc"], pos=wellName, jID=jobIdx + ) try: - _infoBox.value = _txtForInfoBox + _cancelJobBtn.text = "Cancel Job {jID} ({posName})".format(jID=jobIdx, posName=wellName) + _cancelJobBtn.enabled = True + _infoBox.setText(_txtForInfoBox) except: # deleted by user - no longer needs updating - self.results[expIdx]["status"] = STATUS_user_cleared_job + params["status"] = STATUS_user_cleared_job return _tUpdateCount = 0 - _tUpdateCountTimeout = 120 # 2 mins + _tUpdateCountTimeout = ( + jobs_mgmt.JOBS_TIMEOUT * 60 + ) # 5 mins - match executor time-out _lastUpdate_jobTXT = "" + jobTXT = "" while True: - time.sleep(1) # update every sec and exit on break + time.sleep(1) # update every sec and exit on break try: + if "cancel called" in _cancelJobBtn.text: + json_obj = {"uID":expIdx, "jID":jobIdx, "command":"cancel"} + json_str = json.dumps(json_obj)+"\n" + client_socket.send(json_str.encode()) + params["status"] = STATUS_user_cancelled_job + _infoBox.setText( + "User called for Cancel Job Request\n" + + "Please check terminal output for Job status..\n\n" + + jobTXT + ) + self.clientRelease(expIdx, jobIdx, client_socket, params) + break # cancel called by user if _infoBox == None: - self.results[expIdx]["status"] = STATUS_user_cleared_job - break # deleted by user - no longer needs updating - if _infoBox.value: + params["status"] = STATUS_user_cleared_job + self.clientRelease(expIdx, jobIdx, client_socket, params) + break # deleted by user - no longer needs updating + if _infoBox: pass - except: - self.results[expIdx]["status"] = STATUS_user_cleared_job - break # deleted by user - no longer needs updating - if self.JobsMgmt.hasSubmittedJob(expIdx): + except Exception as exc: + print(exc.args) + params["status"] = STATUS_user_cleared_job + self.clientRelease(expIdx, jobIdx, client_socket, params) + break # deleted by user - no longer needs updating + if self.JobsMgmt.hasSubmittedJob(expIdx, jobIdx, mode="server"): if params["status"] in [STATUS_finished_job]: + self.clientRelease(expIdx, jobIdx, client_socket, params) break elif params["status"] in [STATUS_errored_job]: - jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") - _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobERR + jobERR = self.JobsMgmt.checkForJobIDFile( + jobIdx, extension="err" + ) + _infoBox.setText( + jobIdx + "\n" + params["desc"] + "\n\n" + jobERR + ) + self.clientRelease(expIdx, jobIdx, client_socket, params) break else: - jobTXT = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="out") + jobTXT = self.JobsMgmt.checkForJobIDFile( + jobIdx, extension="out" + ) try: - if jobTXT == "": # job file not created yet + if jobTXT == "": # job file not created yet time.sleep(2) - elif self.results[expIdx]["status"] == STATUS_finished_job: + elif ( + params["status"] + == STATUS_finished_job + ): rowIdx = self.findWidgetRowInLayout(expIdx) # check to ensure row deletion due to shrinking table # if not deleted try to delete again if rowIdx < 0: + self.clientRelease(expIdx, jobIdx, client_socket, params) break else: ROW_POP_QUEUE.append(expIdx) elif JOB_COMPLETION_STR in jobTXT: - self.results[expIdx]["status"] = STATUS_finished_job - _infoBox.value = jobTXT - # this is the only case where row deleting occurs + params["status"] = STATUS_finished_job + _infoBox.setText(jobTXT) + # this is the only case where row deleting occurs # we cant delete the row directly from this thread # we will use the exp_id to identify and delete the row # using pyqtSignal - ROW_POP_QUEUE.append(expIdx) + ROW_POP_QUEUE.append(expIdx) # break - based on status elif JOB_TRIGGERED_EXC in jobTXT: - self.results[expIdx]["status"] = STATUS_errored_job - jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") - _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobTXT +"\n\n"+ jobERR + params["status"] = STATUS_errored_job + jobERR = self.JobsMgmt.checkForJobIDFile( + jobIdx, extension="err" + ) + _infoBox.setText( + jobIdx + + "\n" + + params["desc"] + + "\n\n" + + jobTXT + + "\n\n" + + jobERR + ) + self.clientRelease(expIdx, jobIdx, client_socket, params) break elif JOB_RUNNING_STR in jobTXT: - self.results[expIdx]["status"] = STATUS_running_job - _infoBox.value = jobTXT + params["status"] = STATUS_running_job + _infoBox.setText(jobTXT) _tUpdateCount += 1 - if _tUpdateCount > 60: + if _tUpdateCount > 60: if _lastUpdate_jobTXT != jobTXT: # if there is an update reset counter - _tUpdateCount=0 + _tUpdateCount = 0 _lastUpdate_jobTXT = jobTXT else: - _infoBox.value = "Please check terminal output for Job status..\n\n" + jobTXT + _infoBox.setText( + "Please check terminal output for Job status..\n\n" + + jobTXT + ) if _tUpdateCount > _tUpdateCountTimeout: - break + self.clientRelease(expIdx, jobIdx, client_socket, params) + break else: - jobERR = self.JobsMgmt.checkForJobIDFile(jobIdx, extension="err") - _infoBox.value = jobIdx + "\n" + params["desc"] +"\n\n"+ jobERR + jobERR = self.JobsMgmt.checkForJobIDFile( + jobIdx, extension="err" + ) + _infoBox.setText( + jobIdx + + "\n" + + params["desc"] + + "\n\n" + + jobERR + ) + self.clientRelease(expIdx, jobIdx, client_socket, params) break except Exception as exc: print(exc.args) + else: + self.clientRelease(expIdx, jobIdx, client_socket, params) + break else: # this would occur when an exception happens on the pool side before or during job submission - # we dont have a job ID and will update based on exp_ID/uIU - for param_ID in self.results.keys(): - params = self.results[param_ID] + # we dont have a job ID and will update based on exp_ID/uID + # if job submission was not successful we can assume the client is not listening + # and does not require a clientRelease cmd + for uID in self.results.keys(): + params = self.results[uID]["JobUNK"] if params["status"] in [STATUS_errored_pool]: _infoBox = params["table_entry_infoBox"] - poolERR = self.results[params["exp_id"]]["error"] - _infoBox.value = poolERR + poolERR = params["error"] + _infoBox.setText(poolERR) + + def clientRelease(self, expIdx, jobIdx, client_socket, params): + # only need to release client from primary job + self.JobsMgmt.putJobCompletionInList(True, expIdx, jobIdx) + if params["primary"]: + while not self.JobsMgmt.checkAllExpJobsCompletion(expIdx): + time.sleep(1) + json_obj = {"uID":expIdx, "jID":jobIdx,"command": "clientRelease"} + json_str = json.dumps(json_obj)+"\n" + client_socket.send(json_str.encode()) def runInPool(self, params): - self.results[params["exp_id"]] = params - self.results[params["exp_id"]]["status"] = STATUS_running_pool - self.results[params["exp_id"]]["error"] = "" - try: + self.results[params["exp_id"]] = {} + self.results[params["exp_id"]]["JobUNK"] = params + self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_running_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = "" + try: self.pool.submit(self.run, params) except Exception as exc: - self.results[params["exp_id"]]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) + self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() def runMultiInPool(self, multi_params_as_list): for params in multi_params_as_list: - self.results[params["exp_id"]] = params - self.results[params["exp_id"]]["status"] = STATUS_submitted_pool - self.results[params["exp_id"]]["error"] = "" - try: + self.results[params["exp_id"]] = {} + self.results[params["exp_id"]]["JobUNK"] = params + self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_submitted_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = "" + try: self.pool.map(self.run, multi_params_as_list) except Exception as exc: for params in multi_params_as_list: - self.results[params["exp_id"]]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) + self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = str( + "\n".join(exc.args) + ) self.tableUpdateAndCleaupThread() def getResults(self): return self.results - + def getResult(self, exp_id): return self.results[exp_id] - - def run(self, params): - # thread where work is passed to CLI which will handle the + + def run(self, params): + # thread where work is passed to CLI which will handle the # multi-processing aspects based on resources if params["exp_id"] not in self.results.keys(): - self.results[params["exp_id"]] = params - self.results[params["exp_id"]]["error"] = "" + self.results[params["exp_id"]] = {} + self.results[params["exp_id"]]["JobUNK"] = params + self.results[params["exp_id"]]["JobUNK"]["error"] = "" try: # does need further threading ? probably not ! - thread = threading.Thread(target=self.runInSubProcess, args=(params,)) + thread = threading.Thread( + target=self.runInSubProcess, args=(params,) + ) thread.start() - + # self.runInSubProcess(params) # check for this job to show up in submitit jobs list # wait for 2 sec before raising an error except Exception as exc: - self.results[params["exp_id"]]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) + self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() def runInSubProcess(self, params): - try: + try: input_path = str(params["input_path"]) config_path = str(params["config_path"]) output_path = str(params["output_path"]) - uid = str(params["exp_id"]) + uid = str(params["exp_id"]) mainfp = str(main.FILE_PATH) - self.results[params["exp_id"]]["status"] = STATUS_submitted_job - - proc = subprocess.run(['python', mainfp, 'reconstruct', '-i', input_path, '-c', config_path, '-o', output_path, '-uid', uid]) - self.results[params["exp_id"]]["proc"] = proc + self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_submitted_job + + proc = subprocess.run( + [ + "python", + mainfp, + "reconstruct", + "-i", + input_path, + "-c", + config_path, + "-o", + output_path, + "-uid", + uid, + ] + ) + self.results[params["exp_id"]]["JobUNK"]["proc"] = proc if proc.returncode != 0: - raise Exception("An error occurred in processing ! Check terminal output.") + raise Exception( + "An error occurred in processing ! Check terminal output." + ) + + except Exception as exc: + self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) + self.tableUpdateAndCleaupThread() + +class AddWidgetWorkerThread(QThread): + add_widget_signal = pyqtSignal(QVBoxLayout, str, str, str, str) + + def __init__(self, layout, expID, jID, desc, wellName): + super().__init__() + self.layout = layout + self.expID = expID + self.jID = jID + self.desc = desc + self.wellName = wellName - except Exception as exc: - self.results[params["exp_id"]]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["error"] = str("\n".join(exc.args)) - self.tableUpdateAndCleaupThread() + def run(self): + # Emit the signal to add the widget to the main thread + self.add_widget_signal.emit(self.layout, self.expID, self.jID, self.desc, self.wellName) ROW_POP_QUEUE = [] + + # Emits a signal to QFormLayout on the main thread class RowDeletionWorkerThread(QThread): removeRowSignal = pyqtSignal(int, str) @@ -1799,7 +2487,7 @@ def __init__(self, formLayout): self.formLayout = formLayout def setNewInstances(self, formLayout): - self.formLayout:QFormLayout = formLayout + self.formLayout: QFormLayout = formLayout # we might deal with race conditions with a shrinking table # find out widget and return its index @@ -1827,8 +2515,65 @@ def run(self): else: time.sleep(5) +class DropButton(QPushButton): + def __init__(self, text, parent=None, recon_tab:Ui_ReconTab_Form=None): + super().__init__(text, parent) + self.setAcceptDrops(True) + self.recon_tab = recon_tab + + def dragEnterEvent(self, event): + if event.mimeData().hasUrls(): + event.acceptProposedAction() + + def dropEvent(self, event): + files = [] + for url in event.mimeData().urls(): + filepath = url.toLocalFile() + files.append(filepath) + self.recon_tab.openModelFiles(files) + +class DropWidget(QWidget): + def __init__(self, recon_tab:Ui_ReconTab_Form=None): + super().__init__() + self.setAcceptDrops(True) + self.recon_tab = recon_tab + + def dragEnterEvent(self, event): + if event.mimeData().hasUrls(): + event.acceptProposedAction() + + def dropEvent(self, event): + files = [] + for url in event.mimeData().urls(): + filepath = url.toLocalFile() + files.append(filepath) + self.recon_tab.openModelFiles(files) + +class ScrollableLabel(QScrollArea): + def __init__(self, text, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.label = QLabel() + self.label.setWordWrap(True) + self.label.setText(text) + + layout = QVBoxLayout() + layout.setAlignment(Qt.AlignmentFlag.AlignTop) + layout.addWidget(self.label) + + container = QWidget() + container.setLayout(layout) + + self.setWidget(container) + self.setWidgetResizable(True) + + def setText(self, text): + self.label.setText(text) + + # VScode debugging if __name__ == "__main__": import napari + napari.Viewer() - napari.run() \ No newline at end of file + napari.run() diff --git a/recOrder/tests/widget_tests/test_pydantic_model_widget.py b/recOrder/tests/widget_tests/test_pydantic_model_widget.py index 7f3a9bd0..332e6cff 100644 --- a/recOrder/tests/widget_tests/test_pydantic_model_widget.py +++ b/recOrder/tests/widget_tests/test_pydantic_model_widget.py @@ -11,7 +11,7 @@ class MainWindow(QWidget): def __init__(self): super().__init__() - recon_tab = tab_recon.Ui_ReconTab_Form() + recon_tab = tab_recon.Ui_ReconTab_Form(stand_alone=True) layout = QVBoxLayout() self.setLayout(layout) layout.addWidget(recon_tab.recon_tab_widget) From dee21447848c0306bc3fb58ae8c24b72a2d34a27 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Fri, 27 Dec 2024 16:39:31 -0500 Subject: [PATCH 14/38] - fixes & enhancements - stand-alone GUI cmd - refactored UI based on suggestions - fixes cyclic import for stand-alone GUI - duplicates prev model settings when available - clears Model when changing Input datasets - load model looks for .yml files --- recOrder/cli/gui_widget.py | 41 ++++++++ recOrder/cli/jobs_mgmt.py | 3 + recOrder/cli/main.py | 7 +- recOrder/plugin/tab_recon.py | 178 +++++++++++++++++++++-------------- 4 files changed, 154 insertions(+), 75 deletions(-) create mode 100644 recOrder/cli/gui_widget.py diff --git a/recOrder/cli/gui_widget.py b/recOrder/cli/gui_widget.py new file mode 100644 index 00000000..14f68872 --- /dev/null +++ b/recOrder/cli/gui_widget.py @@ -0,0 +1,41 @@ +import sys +from PyQt6.QtWidgets import QApplication, QWidget, QVBoxLayout, QStyle +import click +from recOrder.plugin import tab_recon + +try: + import qdarktheme +except:pass + +PLUGIN_NAME = "recOrder: Computational Toolkit for Label-Free Imaging" +PLUGIN_ICON = "🔬" + +@click.command() +def gui(): + """GUI for recOrder: Computational Toolkit for Label-Free Imaging""" + + app = QApplication(sys.argv) + app.setStyle("Fusion") # Other options: "Fusion", "Windows", "macOS", "WindowsVista" + try: + qdarktheme.setup_theme('auto') + except:pass + window = MainWindow() + window.setWindowTitle(PLUGIN_ICON + " " + PLUGIN_NAME + " " + PLUGIN_ICON) + + pixmapi = getattr(QStyle.StandardPixmap, "SP_TitleBarMenuButton") + icon = app.style().standardIcon(pixmapi) + window.setWindowIcon(icon) + + window.show() + sys.exit(app.exec()) + +class MainWindow(QWidget): + def __init__(self): + super().__init__() + recon_tab = tab_recon.Ui_ReconTab_Form(stand_alone=True) + layout = QVBoxLayout() + self.setLayout(layout) + layout.addWidget(recon_tab.recon_tab_widget) + +if __name__ == '__main__': + gui() \ No newline at end of file diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py index 96af8630..d362487b 100644 --- a/recOrder/cli/jobs_mgmt.py +++ b/recOrder/cli/jobs_mgmt.py @@ -3,6 +3,9 @@ import submitit import threading, time +DIR_PATH = os.path.dirname(os.path.realpath(__file__)) +FILE_PATH = os.path.join(DIR_PATH, "main.py") + SERVER_PORT = 8089 # Choose an available port JOBS_TIMEOUT = 5 # 5 mins SERVER_uIDsjobIDs = {} # uIDsjobIDs[uid][jid] = job diff --git a/recOrder/cli/main.py b/recOrder/cli/main.py index f4c51382..b1b7a32b 100644 --- a/recOrder/cli/main.py +++ b/recOrder/cli/main.py @@ -1,14 +1,13 @@ -import click, os +import click from recOrder.cli.apply_inverse_transfer_function import apply_inv_tf from recOrder.cli.compute_transfer_function import compute_tf from recOrder.cli.reconstruct import reconstruct +from recOrder.cli.gui_widget import gui CONTEXT = {"help_option_names": ["-h", "--help"]} -DIR_PATH = os.path.dirname(os.path.realpath(__file__)) -FILE_PATH = os.path.join(DIR_PATH, "main.py") # `recorder -h` will show subcommands in the order they are added class NaturalOrderGroup(click.Group): @@ -24,7 +23,7 @@ def cli(): cli.add_command(reconstruct) cli.add_command(compute_tf) cli.add_command(apply_inv_tf) - +cli.add_command(gui) if __name__ == '__main__': cli() \ No newline at end of file diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index d642cb62..a27ee4b4 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -15,7 +15,7 @@ import warnings from recOrder.io import utils -from recOrder.cli import settings, main, jobs_mgmt +from recOrder.cli import settings, jobs_mgmt from napari.utils import notifications from concurrent.futures import ThreadPoolExecutor @@ -113,12 +113,14 @@ def __init__(self, parent=None, stand_alone=False): self.stand_alone = stand_alone if HAS_INSTANCE["val"]: self.current_dir_path = str(Path.cwd()) + self.directory = str(Path.cwd()) self.current_save_path = HAS_INSTANCE["current_save_path"] self.input_directory = HAS_INSTANCE["input_directory"] self.save_directory = HAS_INSTANCE["save_directory"] self.model_directory = HAS_INSTANCE["model_directory"] self.yaml_model_file = HAS_INSTANCE["yaml_model_file"] else: + self.directory = str(Path.cwd()) self.current_dir_path = str(Path.cwd()) self.current_save_path = str(Path.cwd()) self.input_directory = str(Path.cwd()) @@ -145,23 +147,52 @@ def __init__(self, parent=None, stand_alone=False): self.modes_widget2.setMaximumHeight(50) self.modes_widget2.setMinimumHeight(50) + self.reconstruction_input_data_label = widgets.Label( + name="", value="Input Data" + ) self.reconstruction_input_data_loc = widgets.LineEdit( name="", value=self.input_directory ) self.reconstruction_input_data_btn = widgets.PushButton( - name="InputData", label="Input Data" + name="InputData", label="Browse" ) + self.reconstruction_input_data_btn.native.setMinimumWidth(125) self.reconstruction_input_data_btn.clicked.connect( self.browse_dir_path_input ) self.reconstruction_input_data_loc.changed.connect( self.readAndSetInputPathOnValidation ) + _load_model_btn = DropButton(text="Load Model(s)", recon_tab=self) + _load_model_btn.setMinimumWidth(125) + self.modes_layout2.addWidget(self.reconstruction_input_data_label.native) self.modes_layout2.addWidget(self.reconstruction_input_data_loc.native) self.modes_layout2.addWidget(self.reconstruction_input_data_btn.native) + self.modes_layout2.addWidget(_load_model_btn) self.recon_tab_layout.addWidget(self.modes_widget2) + # _load_model_label = widgets.Label(name="", value="Models Path") + # _load_model_loc = widgets.LineEdit(name="", value=self.model_directory) + + # Passing model location label to model location selector + _load_model_btn.clicked.connect( + lambda: self.browse_dir_path_model() + ) + + # HBox for Loading Model + # _hBox_widget_model = QWidget() + # _hBox_layout_model = QHBoxLayout() + # _hBox_layout_model.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + # _hBox_widget_model.setLayout(_hBox_layout_model) + # _hBox_widget_model.setMaximumHeight(50) + # _hBox_widget_model.setMinimumHeight(50) + # _hBox_layout_model.addWidget(_load_model_label.native) + # _hBox_layout_model.addWidget(_load_model_loc.native) + # _hBox_layout_model.addWidget(_load_model_btn) + + # self.recon_tab_layout.addWidget(_hBox_widget_model) + # Top level - Selection modes, model creation and running self.modes_widget = QWidget() self.modes_layout = QHBoxLayout() @@ -187,59 +218,35 @@ def __init__(self, parent=None, stand_alone=False): # PushButton to create a copy of the model - UI self.reconstruction_mode_enabler = widgets.PushButton( - name="CreateModel", label="Create Model" + name="BuildModel", label="Build Model" ) + self.reconstruction_mode_enabler.native.setMinimumWidth(125) self.reconstruction_mode_enabler.clicked.connect( - self._create_acq_contols - ) - - # PushButton to validate and create the yaml file(s) based on selection - self.build_button = widgets.PushButton(name="Build && Run Model") - self.build_button.clicked.connect(self.build_model_and_run) + self._build_acq_contols + ) # PushButton to clear all copies of models that are create for UI self.reconstruction_mode_clear = widgets.PushButton( name="ClearModels", label="Clear All Models" ) + self.reconstruction_mode_clear.native.setMinimumWidth(125) self.reconstruction_mode_clear.clicked.connect(self._clear_all_models) + # PushButton to validate and create the yaml file(s) based on selection + self.build_button = widgets.PushButton(name="Run Model") + self.build_button.native.setMinimumWidth(125) + self.build_button.clicked.connect(self.build_model_and_run) + # Editable List holding pydantic class(es) as per user selection self.pydantic_classes = list() + self.prev_model_settings = {} self.index = 0 - self.modes_layout.addWidget(self.reconstruction_mode_enabler.native) - self.modes_layout.addWidget(self.build_button.native) + self.modes_layout.addWidget(self.reconstruction_mode_enabler.native) self.modes_layout.addWidget(self.reconstruction_mode_clear.native) + self.modes_layout.addWidget(self.build_button.native) self.recon_tab_layout.addWidget(self.modes_widget) - - _load_model_loc = widgets.LineEdit(name="", value=self.model_directory) - # _load_model_btn = widgets.PushButton( - # name="LoadModel", label="Load Model" - # ) - _load_model_btn = DropButton(text="Load Model", recon_tab=self) - - # Passing model location label to model location selector - _load_model_btn.clicked.connect( - lambda: self.browse_dir_path_model(_load_model_loc) - ) - - _clear_results_btn = widgets.PushButton( - name="ClearResults", label="Clear Results" - ) - _clear_results_btn.clicked.connect(self.clear_results_table) - - # HBox for Loading Model - _hBox_widget_model = QWidget() - _hBox_layout_model = QHBoxLayout() - _hBox_layout_model.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - _hBox_widget_model.setLayout(_hBox_layout_model) - _hBox_widget_model.setMaximumHeight(50) - _hBox_widget_model.setMinimumHeight(50) - _hBox_layout_model.addWidget(_load_model_loc.native) - _hBox_layout_model.addWidget(_load_model_btn) - _hBox_layout_model.addWidget(_clear_results_btn.native) - self.recon_tab_layout.addWidget(_hBox_widget_model) - + # Line seperator between top / middle UI components _line = QFrame() _line.setMinimumWidth(1) @@ -268,10 +275,9 @@ def __init__(self, parent=None, stand_alone=False): ) # Create the splitter - splitter = QSplitter(self) + splitter = QSplitter() splitter.setOrientation(Qt.Orientation.Vertical) splitter.setSizes([600, 200]) - self.recon_tab_layout.addWidget(splitter) # self.recon_tab_layout.addWidget(self.recon_tab_scrollArea_settings) @@ -292,9 +298,10 @@ def __init__(self, parent=None, stand_alone=False): _scrollArea.setWidget(_qwidget_settings) splitter.addWidget(_scrollArea) - my_splitter_handle = splitter.handle(1) + my_splitter_handle = splitter.handle(1) my_splitter_handle.setStyleSheet("background: 1px rgb(128,128,128);") splitter.setStyleSheet("""QSplitter::handle:pressed {background-color: #ca5;}""") + # Table for processing entries self.proc_table_QFormLayout = QFormLayout() @@ -306,6 +313,12 @@ def __init__(self, parent=None, stand_alone=False): QSizePolicy.Expanding, QSizePolicy.Expanding ) _proc_table_widget.setLayout(self.proc_table_QFormLayout) + + _clear_results_btn = widgets.PushButton( + name="ClearResults", label="Clear Results" + ) + _clear_results_btn.clicked.connect(self.clear_results_table) + _qwidget_settings_layout.addWidget(_clear_results_btn.native) _qwidget_settings_layout.addWidget(_proc_table_widget) # Stores Model & Components values which cause validation failure - can be highlighted on the model field as Red @@ -350,12 +363,12 @@ def showEvent(self, event): if event.type() == QEvent.Type.Show: pass - def confirmDialog(self): + def confirmDialog(self, msg="Confirm your selection ?"): qm = QMessageBox ret = qm.question( self.recon_tab_widget, "Confirm", - "Confirm your selection ?", + msg, qm.Yes | qm.No, ) if ret == qm.Yes: @@ -367,6 +380,10 @@ def confirmDialog(self): # ToDo: utilize common functions # Input data selector def browse_dir_path_input(self): + if len(self.pydantic_classes)>0 and not self.confirmDialog("Changing Input Data will reset your models. Continue ?"): + return + else: + self._clear_all_models(silent=True) result = self._open_file_dialog(self.input_directory, "dir") if result == "": return @@ -376,11 +393,13 @@ def browse_dir_path_input(self): self.messageBox(ret_msg) return - self.directory = result + self.directory = Path(result).parent.absolute() self.current_dir_path = result self.input_directory = result self.reconstruction_input_data_loc.value = result + self.prev_model_settings = {} + self.saveLastPaths() def browse_dir_path_inputBG(self, elem): @@ -434,7 +453,7 @@ def readAndSetInputPathOnValidation(self): valid, ret_msg = self.validateInputData(result) if valid: - self.directory = result + self.directory = Path(result).parent.absolute() self.current_dir_path = result self.input_directory = result @@ -479,9 +498,9 @@ def readAndSetOutputPathOnValidation(self, elem1, elem2, save_path): # Copied from main_widget # ToDo: utilize common functions # Output data selector - def browse_dir_path_model(self, elem): + def browse_dir_path_model(self): results = self._open_file_dialog( - self.model_directory, "files" + self.directory, "files", filter="YAML Files (*.yml)" ) # returns list if len(results) == 0 or results == "": return @@ -490,11 +509,7 @@ def browse_dir_path_model(self, elem): self.directory = self.model_directory self.current_dir_path = self.model_directory - elem.value = str(Path(results[0]).absolute()) - if len(results) > 1: - elem.value = self.model_directory self.saveLastPaths() - self.openModelFiles(results) @@ -597,6 +612,10 @@ def saveLastPaths(self): # clears the results table def clear_results_table(self): + index = self.proc_table_QFormLayout.rowCount() + if index < 1: + self.messageBox("There are no processing results to clear !") + return if self.confirmDialog(): for i in range(self.proc_table_QFormLayout.rowCount()): self.proc_table_QFormLayout.removeRow(0) @@ -807,7 +826,7 @@ def messageBox(self, msg, type="exc"): notifications.show_info(json_txt) def messageBoxStandAlone(self, msg): - q = QMessageBox(QMessageBox.Warning, "Message", str(msg)) + q = QMessageBox(QMessageBox.Warning, "Message", str(msg), parent=self.recon_tab_widget) q.setStandardButtons(QMessageBox.StandardButton.Ok) q.setIcon(QMessageBox.Icon.Warning) q.exec_() @@ -1039,7 +1058,7 @@ def _fix_model(self, model, exclude_modes, attr_key, attr_val): return model # Creates UI controls from model based on selections - def _create_acq_contols(self): + def _build_acq_contols(self): # Make a copy of selections and unsed for deletion selected_modes = [] @@ -1057,6 +1076,17 @@ def _create_acq_contols(self): def _create_acq_contols2( self, selected_modes, exclude_modes, myLoadedModel=None, json_dict=None ): + # duplicate settings from the prev model on new model creation + if json_dict is None and len(self.pydantic_classes) > 0: + ret = self.build_model_and_run(validate_return_prev_model_json_txt=True) + if ret is None: + return + key, json_txt = ret + self.prev_model_settings[key] = json.loads(json_txt) + if json_dict is None: + key = "-".join(selected_modes) + if key in self.prev_model_settings.keys(): + json_dict = self.prev_model_settings[key] # initialize the top container and specify what pydantic class to map from if myLoadedModel is not None: @@ -1171,7 +1201,7 @@ def _create_acq_contols2( # These could be multiple based on user selection for each model # Inherits from Input by default at creation time name_without_ext = os.path.splitext(self.input_directory)[0] - save_path = os.path.join(Path(self.input_directory).parent.absolute(), (name_without_ext + ("_recon"+c_mode_short+num_str) + ".zarr")) + save_path = os.path.join(Path(self.input_directory).parent.absolute(), (name_without_ext + ("_"+c_mode_short+"_"+num_str) + ".zarr")) save_path_exists = True if Path(save_path).exists() else False _output_data_loc = widgets.LineEdit( name="", value=save_path, tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") @@ -1283,11 +1313,11 @@ def _create_acq_contols2( self.index += 1 if self.index > 1: - self.build_button.text = "Build && Run {n} Models".format( + self.build_button.text = "Run {n} Models".format( n=self.index ) else: - self.build_button.text = "Build && Run Model" + self.build_button.text = "Run Model" return pydantic_model @@ -1319,15 +1349,16 @@ def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, index, _str): i += 1 self.index = len(self.pydantic_classes) if self.index > 1: - self.build_button.text = "Build && Run {n} Models".format( + self.build_button.text = "Run {n} Models".format( n=self.index ) else: - self.build_button.text = "Build && Run Model" + self.build_button.text = "Run Model" # Clear all the generated pydantic models and clears the pydantic model list - def _clear_all_models(self): - if self.confirmDialog(): + def _clear_all_models(self, silent=False): + + if silent or self.confirmDialog(): index = self.recon_tab_qwidget_settings_layout.count() - 1 while index >= 0: myWidget = self.recon_tab_qwidget_settings_layout.itemAt( @@ -1339,11 +1370,12 @@ def _clear_all_models(self): self.pydantic_classes.clear() CONTAINERS_INFO.clear() self.index = 0 - self.build_button.text = "Build && Run Model" + self.build_button.text = "Run Model" + self.prev_model_settings = None # Displays the json output from the pydantic model UI selections by user # Loops through all our stored pydantic classes - def build_model_and_run(self): + def build_model_and_run(self, validate_return_prev_model_json_txt=False): # we dont want to have a partial run if there are N models # so we will validate them all first and then run in a second loop # first pass for validating @@ -1446,6 +1478,9 @@ def build_model_and_run(self): fmt_str = self.formatStringForErrorDisplay(_collectAllErrors) self.messageBox(fmt_str) return + + if validate_return_prev_model_json_txt: + return "-".join(selected_modes), json_txt # generate a time-stamp for our yaml files to avoid overwriting # files generated at the same time will have an index suffix @@ -1898,7 +1933,7 @@ def get_pydantic_kwargs( # copied from main_widget # file open/select dialog - def _open_file_dialog(self, default_path, type): + def _open_file_dialog(self, default_path, type, filter="All Files (*)"): if type == "dir": return self._open_dialog( "select a directory", str(default_path), type @@ -1906,7 +1941,7 @@ def _open_file_dialog(self, default_path, type): elif type == "file": return self._open_dialog("select a file", str(default_path), type) elif type == "files": - return self._open_dialog("select file(s)", str(default_path), type) + return self._open_dialog("select file(s)", str(default_path), type, filter) elif type == "save": return self._open_dialog("save a file", str(default_path), type) else: @@ -1914,7 +1949,7 @@ def _open_file_dialog(self, default_path, type): "select a directory", str(default_path), type ) - def _open_dialog(self, title, ref, type): + def _open_dialog(self, title, ref, type, filter="All Files (*)"): """ opens pop-up dialogue for the user to choose a specific file or directory. @@ -1940,7 +1975,7 @@ def _open_dialog(self, title, ref, type): )[0] elif type == "files": path = QFileDialog.getOpenFileNames( - None, title, ref, options=options + None, title, ref, filter=filter, options=options )[0] elif type == "save": path = QFileDialog.getSaveFileName( @@ -2430,7 +2465,7 @@ def runInSubProcess(self, params): config_path = str(params["config_path"]) output_path = str(params["output_path"]) uid = str(params["exp_id"]) - mainfp = str(main.FILE_PATH) + mainfp = str(jobs_mgmt.FILE_PATH) self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_submitted_job @@ -2445,8 +2480,10 @@ def runInSubProcess(self, params): config_path, "-o", output_path, + "-rx", + str(20), "-uid", - uid, + uid ] ) self.results[params["exp_id"]]["JobUNK"]["proc"] = proc @@ -2570,7 +2607,6 @@ def __init__(self, text, *args, **kwargs): def setText(self, text): self.label.setText(text) - # VScode debugging if __name__ == "__main__": import napari From d82c13a01681bcc1f2438bb3ffe61f529d5198fb Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sat, 28 Dec 2024 01:47:07 -0500 Subject: [PATCH 15/38] checking for RuntimeWarning value --- recOrder/tests/util_tests/test_overlays.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recOrder/tests/util_tests/test_overlays.py b/recOrder/tests/util_tests/test_overlays.py index d483a26e..8354bd69 100644 --- a/recOrder/tests/util_tests/test_overlays.py +++ b/recOrder/tests/util_tests/test_overlays.py @@ -20,7 +20,7 @@ def _birefringence(draw): dtype, shape=shape, elements=st.floats( - min_value=0, + min_value=1.0000000168623835e-16, max_value=50, exclude_min=True, width=bit_width, @@ -40,6 +40,7 @@ def _birefringence(draw): ), ) ) + return retardance, orientation From 96709e108ecb44e92bf14b7322cb8a963a9c9c05 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 7 Jan 2025 02:38:08 -0500 Subject: [PATCH 16/38] on-the-fly processing - checks if data is being acquired and utilizes polling (every 10s) to gather processing information and submits processing jobs once a minimum dim has been acquired - added a script to simulate acq of .zarr storage - added group-box for sections - added scrolling for tab so that it does not block horizontal resizing --- recOrder/acq/acquisition_workers.py | 1 + recOrder/cli/jobs_mgmt.py | 5 +- recOrder/plugin/gui.py | 2 +- recOrder/plugin/tab_recon.py | 802 ++++++++++++++---- .../tests/widget_tests/test_simulate_acq.py | 147 ++++ 5 files changed, 802 insertions(+), 155 deletions(-) create mode 100644 recOrder/tests/widget_tests/test_simulate_acq.py diff --git a/recOrder/acq/acquisition_workers.py b/recOrder/acq/acquisition_workers.py index 4b19ab5e..ffd30517 100644 --- a/recOrder/acq/acquisition_workers.py +++ b/recOrder/acq/acquisition_workers.py @@ -598,6 +598,7 @@ def _reconstruct(self): transfer_function_dirpath=transfer_function_path, config_filepath=self.config_path, output_dirpath=reconstruction_path, + unique_id="recOrderAcq" ) # Read reconstruction to pass to emitters diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py index d362487b..fc0ed07a 100644 --- a/recOrder/cli/jobs_mgmt.py +++ b/recOrder/cli/jobs_mgmt.py @@ -21,7 +21,7 @@ def clearLogs(self): thread = threading.Thread(target=self.clearLogFiles, args={self.logsPath,}) thread.start() - def clearLogFiles(self, dirPath): + def clearLogFiles(self, dirPath, silent=True): for filename in os.listdir(dirPath): file_path = os.path.join(dirPath, filename) try: @@ -30,7 +30,8 @@ def clearLogFiles(self, dirPath): elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception as e: - print('Failed to delete %s. Reason: %s' % (file_path, e)) + if not silent: + print('Failed to delete %s. Reason: %s' % (file_path, e)) def checkForJobIDFile(self, jobID, extension="out"): files = os.listdir(self.logsPath) diff --git a/recOrder/plugin/gui.py b/recOrder/plugin/gui.py index c2f32797..f9e53cb0 100644 --- a/recOrder/plugin/gui.py +++ b/recOrder/plugin/gui.py @@ -926,7 +926,7 @@ def setupUi(self, Form): self.tabWidget.addTab(self.Acquisition, "") self.recon_tab = tab_recon.Ui_ReconTab_Form(Form) - self.tabWidget.addTab(self.recon_tab.recon_tab_widget, 'Reconstruction') + self.tabWidget.addTab(self.recon_tab.recon_tab_mainScrollArea, 'Reconstruction') self.Display = QtWidgets.QWidget() self.Display.setObjectName("Display") diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index a27ee4b4..e3f835a4 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1,4 +1,5 @@ import os, json, subprocess, time, datetime, uuid +import socket, threading from pathlib import Path from qtpy import QtCore @@ -18,7 +19,7 @@ from recOrder.cli import settings, jobs_mgmt from napari.utils import notifications -from concurrent.futures import ThreadPoolExecutor +import concurrent.futures import importlib.metadata @@ -81,6 +82,8 @@ _validate_alert = "⚠" _validate_ok = "✔️" +_green_dot = "🟢" +_red_dot = "🔴" # For now replicate CLI processing modes - these could reside in the CLI settings file as well # for consistency @@ -131,7 +134,7 @@ def __init__(self, parent=None, stand_alone=False): self.input_directory_dataset = None self.input_directory_datasetMeta = None - # Top level parent + # Top level parent self.recon_tab_widget = QWidget() self.recon_tab_layout = QVBoxLayout() self.recon_tab_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) @@ -139,6 +142,10 @@ def __init__(self, parent=None, stand_alone=False): self.recon_tab_layout.setSpacing(0) self.recon_tab_widget.setLayout(self.recon_tab_layout) + self.recon_tab_mainScrollArea = QScrollArea() + self.recon_tab_mainScrollArea.setWidgetResizable(True) + self.recon_tab_mainScrollArea.setWidget(self.recon_tab_widget) + # Top level - Data Input self.modes_widget2 = QWidget() self.modes_layout2 = QHBoxLayout() @@ -156,7 +163,7 @@ def __init__(self, parent=None, stand_alone=False): self.reconstruction_input_data_btn = widgets.PushButton( name="InputData", label="Browse" ) - self.reconstruction_input_data_btn.native.setMinimumWidth(125) + self.reconstruction_input_data_btn.native.setMinimumWidth(75) self.reconstruction_input_data_btn.clicked.connect( self.browse_dir_path_input ) @@ -164,7 +171,7 @@ def __init__(self, parent=None, stand_alone=False): self.readAndSetInputPathOnValidation ) _load_model_btn = DropButton(text="Load Model(s)", recon_tab=self) - _load_model_btn.setMinimumWidth(125) + _load_model_btn.setMinimumWidth(90) self.modes_layout2.addWidget(self.reconstruction_input_data_label.native) self.modes_layout2.addWidget(self.reconstruction_input_data_loc.native) @@ -220,7 +227,7 @@ def __init__(self, parent=None, stand_alone=False): self.reconstruction_mode_enabler = widgets.PushButton( name="BuildModel", label="Build Model" ) - self.reconstruction_mode_enabler.native.setMinimumWidth(125) + self.reconstruction_mode_enabler.native.setMinimumWidth(100) self.reconstruction_mode_enabler.clicked.connect( self._build_acq_contols ) @@ -229,18 +236,19 @@ def __init__(self, parent=None, stand_alone=False): self.reconstruction_mode_clear = widgets.PushButton( name="ClearModels", label="Clear All Models" ) - self.reconstruction_mode_clear.native.setMinimumWidth(125) + self.reconstruction_mode_clear.native.setMinimumWidth(110) self.reconstruction_mode_clear.clicked.connect(self._clear_all_models) # PushButton to validate and create the yaml file(s) based on selection self.build_button = widgets.PushButton(name="Run Model") - self.build_button.native.setMinimumWidth(125) + self.build_button.native.setMinimumWidth(100) self.build_button.clicked.connect(self.build_model_and_run) # Editable List holding pydantic class(es) as per user selection self.pydantic_classes = list() self.prev_model_settings = {} self.index = 0 + self.pollData = False self.modes_layout.addWidget(self.reconstruction_mode_enabler.native) self.modes_layout.addWidget(self.reconstruction_mode_clear.native) @@ -260,6 +268,10 @@ def __init__(self, parent=None, stand_alone=False): self.recon_tab_layout.addWidget(_line) # Top level - Central scrollable component which will hold Editable/(vertical) Expanding UI + group_box_process_models = QGroupBox("Processing Models") + group_box_process_models.setMinimumHeight(200) + group_box_process_models_layout = QHBoxLayout() + group_box_process_models.setLayout(group_box_process_models_layout) self.recon_tab_scrollArea_settings = QScrollArea() # self.recon_tab_scrollArea_settings.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.recon_tab_scrollArea_settings.setWidgetResizable(True) @@ -280,8 +292,8 @@ def __init__(self, parent=None, stand_alone=False): splitter.setSizes([600, 200]) self.recon_tab_layout.addWidget(splitter) - # self.recon_tab_layout.addWidget(self.recon_tab_scrollArea_settings) - splitter.addWidget(self.recon_tab_scrollArea_settings) + group_box_process_models_layout.addWidget(self.recon_tab_scrollArea_settings) + splitter.addWidget(group_box_process_models) _scrollArea = QScrollArea() _scrollArea.setWidgetResizable(True) @@ -304,6 +316,24 @@ def __init__(self, parent=None, stand_alone=False): # Table for processing entries + group_box_OTF = QGroupBox("On-The-Fly Processing Queue") + group_box_OTF_layout = QHBoxLayout() + group_box_OTF.setLayout(group_box_OTF_layout) + self.proc_OTF_table_QFormLayout = QFormLayout() + self.proc_OTF_table_QFormLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.proc_OTF_table_QFormLayout.setSpacing(0) + self.proc_OTF_table_QFormLayout.setContentsMargins(0, 0, 0, 0) + _proc_OTF_table_widget = QWidget() + _proc_OTF_table_widget.setSizePolicy( + QSizePolicy.Expanding, QSizePolicy.Expanding + ) + _proc_OTF_table_widget.setLayout(self.proc_OTF_table_QFormLayout) + group_box_OTF_layout.addWidget(_proc_OTF_table_widget) + group_box_OTF.setMaximumHeight(100) + + group_box_JobResults = QGroupBox("Job Results Processing Queue") + group_box_JobResults_layout = QHBoxLayout() + group_box_JobResults.setLayout(group_box_JobResults_layout) self.proc_table_QFormLayout = QFormLayout() self.proc_table_QFormLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) self.proc_table_QFormLayout.setSpacing(0) @@ -313,13 +343,15 @@ def __init__(self, parent=None, stand_alone=False): QSizePolicy.Expanding, QSizePolicy.Expanding ) _proc_table_widget.setLayout(self.proc_table_QFormLayout) + group_box_JobResults_layout.addWidget(_proc_table_widget) _clear_results_btn = widgets.PushButton( name="ClearResults", label="Clear Results" ) _clear_results_btn.clicked.connect(self.clear_results_table) + _qwidget_settings_layout.addWidget(group_box_OTF) _qwidget_settings_layout.addWidget(_clear_results_btn.native) - _qwidget_settings_layout.addWidget(_proc_table_widget) + _qwidget_settings_layout.addWidget(group_box_JobResults) # Stores Model & Components values which cause validation failure - can be highlighted on the model field as Red self.modelHighlighterVals = {} @@ -384,30 +416,23 @@ def browse_dir_path_input(self): return else: self._clear_all_models(silent=True) - result = self._open_file_dialog(self.input_directory, "dir") - if result == "": + try: + result = self._open_file_dialog(self.input_directory, "dir") + except Exception as exc: + self.messageBox(exc.args) return - ret, ret_msg = self.validateInputData(result) - if not ret: - self.messageBox(ret_msg) + if result == "": return - self.directory = Path(result).parent.absolute() - self.current_dir_path = result - self.input_directory = result - self.reconstruction_input_data_loc.value = result - - self.prev_model_settings = {} - - self.saveLastPaths() + self.reconstruction_input_data_loc.value = result def browse_dir_path_inputBG(self, elem): result = self._open_file_dialog(self.directory, "dir") if result == "": return - ret, ret_msg = self.validateInputData(result) + ret, ret_msg = self.validateInputData(result, BG=True) if not ret: self.messageBox(ret_msg) return @@ -416,7 +441,7 @@ def browse_dir_path_inputBG(self, elem): # not working - not used def validateInputData( - self, input_data_folder: str, metadata=False + self, input_data_folder: str, metadata=False, BG=False ) -> bool: # Sort and validate the input paths, expanding plates into lists of positions # return True, MSG_SUCCESS @@ -425,8 +450,16 @@ def validateInputData( with open_ome_zarr(input_paths, mode="r") as dataset: # ToDo: Metadata reading and implementation in GUI for # channel names, time indicies, etc. - if metadata: + if not BG and metadata: self.input_directory_dataset = dataset + + if not BG: + self.pollData = False + zattrs = dataset.zattrs + if self.isDatasetAcqRunning(zattrs): + if self.confirmDialog(msg="This seems like an in-process Acquisition. Would you like to process data on-the-fly ?"): + self.pollData = True + return True, MSG_SUCCESS raise Exception( "Dataset does not appear to be a valid ome-zarr storage" @@ -452,16 +485,35 @@ def readAndSetInputPathOnValidation(self): result = self.reconstruction_input_data_loc.value valid, ret_msg = self.validateInputData(result) - if valid: + if valid: self.directory = Path(result).parent.absolute() self.current_dir_path = result - self.input_directory = result + self.input_directory = result + + self.prev_model_settings = {} self.saveLastPaths() else: self.reconstruction_input_data_loc.value = self.input_directory self.messageBox(ret_msg) + def isDatasetAcqRunning(self, zattrs: dict)->bool: + """ + Checks the zattrs for CurrentDimensions & FinalDimensions key and tries to figure if + data acquisition is running + """ + + required_order = ['time', 'position', 'z', 'channel'] + if "CurrentDimensions" in zattrs.keys(): + my_dict = zattrs["CurrentDimensions"] + sorted_dict_acq = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} + if "FinalDimensions" in zattrs.keys(): + my_dict = zattrs["FinalDimensions"] + sorted_dict_final = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} + if sorted_dict_acq != sorted_dict_final: + return True + return False + # Copied from main_widget # ToDo: utilize common functions # Output data selector @@ -927,6 +979,22 @@ def addTableEntryJob(self, proc_params): return proc_params + def addRemoveOTFTableEntry(self, OTF_dir_path, bool_msg): + if bool_msg: + otf_label = QLabel(text=OTF_dir_path + " " + _green_dot) + self.proc_OTF_table_QFormLayout.insertRow(0, otf_label) + else: + try: + for row in range(self.proc_OTF_table_QFormLayout.rowCount()): + widgetItem = self.proc_OTF_table_QFormLayout.itemAt(row) + if widgetItem is not None: + name_widget:QLabel = widgetItem.widget() + name_string = str(name_widget.text()) + if OTF_dir_path in name_string: + self.proc_OTF_table_QFormLayout.removeRow(row) + except Exception as exc: + print(exc.args) + # adds processing entry to _qwidgetTabEntry_layout as row item # row item will be purged from table as processing finishes # there could be 3 tabs for this processing table status @@ -1180,13 +1248,13 @@ def _create_acq_contols2( _line.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) _line.setStyleSheet( "border:1px solid rgb(128,128,128); border-width: 1px;" - ) + ) # PushButton to delete a UI container # Use case when a wrong selection of input modes get selected eg Bire+Fl # Preferably this root level validation should occur before values arevalidated # in order to display and avoid this to occur - _del_button = widgets.PushButton(name="Delete this Model") + _del_button = widgets.PushButton(name="Delete Model") c_mode = "-and-".join(selected_modes) c_mode_short = "".join(item[:3].capitalize() for item in selected_modes) @@ -1263,6 +1331,15 @@ def _create_acq_contols2( c_mode_str ) # tableEntryID, tableEntryShortDesc - should update with processing status + _validate_button = widgets.PushButton(name="Validate Model") + _validate_button.clicked.connect(lambda:self._validate_model(_str, _collapsibleBoxWidget)) + + _hBox_widget2 = QWidget() + _hBox_layout2 = QHBoxLayout() + _hBox_layout2.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + _hBox_widget2.setLayout(_hBox_layout2) + _hBox_layout2.addWidget(_validate_button.native) + _hBox_layout2.addWidget(_del_button.native) _expandingTabEntryWidgetLayout = QVBoxLayout() _expandingTabEntryWidgetLayout.addWidget(_collapsibleBoxWidget) @@ -1280,7 +1357,7 @@ def _create_acq_contols2( recon_pydantic_container.native ) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget) - _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_del_button.native) + _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget2) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_line) _scrollAreaCollapsibleBox.setMinimumHeight(_scrollAreaCollapsibleBoxWidgetLayout.sizeHint().height()) @@ -1308,6 +1385,7 @@ def _create_acq_contols2( "container": recon_pydantic_container, "selected_modes": selected_modes.copy(), "exclude_modes": exclude_modes.copy(), + "poll_data": self.pollData, } ) self.index += 1 @@ -1321,6 +1399,106 @@ def _create_acq_contols2( return pydantic_model + def _validate_model(self, _str, _collapsibleBoxWidget): + i = 0 + model_entry_item = None + for item in self.pydantic_classes: + if item["uuid"] == _str: + model_entry_item = item + break + i += 1 + if model_entry_item is not None: + cls = item["class"] + cls_container = item["container"] + exclude_modes = item["exclude_modes"] + c_mode_str = item["c_mode_str"] + + # build up the arguments for the pydantic model given the current container + if cls is None: + self.messageBox("No model defined !") + return + + pydantic_kwargs = {} + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( + cls_container, cls, pydantic_kwargs, exclude_modes + ) + if pydantic_kwargs is None: + self.messageBox(ret_msg) + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_alert}" + ) + return + + input_channel_names, ret_msg = self.clean_string_for_list( + "input_channel_names", pydantic_kwargs["input_channel_names"] + ) + if input_channel_names is None: + self.messageBox(ret_msg) + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_alert}" + ) + return + pydantic_kwargs["input_channel_names"] = input_channel_names + + time_indices, ret_msg = self.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) + if time_indices is None: + self.messageBox(ret_msg) + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_alert}" + ) + return + pydantic_kwargs["time_indices"] = time_indices + + time_indices, ret_msg = self.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) + if time_indices is None: + self.messageBox(ret_msg) + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_alert}" + ) + return + pydantic_kwargs["time_indices"] = time_indices + + if "birefringence" in pydantic_kwargs.keys(): + background_path, ret_msg = self.clean_path_string_when_empty( + "background_path", + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ], + ) + if background_path is None: + self.messageBox(ret_msg) + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_alert}" + ) + return + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ] = background_path + + # validate and return errors if None + pydantic_model, ret_msg = self.validate_pydantic_model( + cls, pydantic_kwargs + ) + if pydantic_model is None: + self.messageBox(ret_msg) + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_alert}" + ) + return + if ret_msg == MSG_SUCCESS: + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_ok}" + ) + else: + _collapsibleBoxWidget.setNewName( + f"{c_mode_str} {_validate_alert}" + ) + + # UI components deletion - maybe just needs the parent container instead of individual components def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, index, _str): @@ -1371,7 +1549,7 @@ def _clear_all_models(self, silent=False): CONTAINERS_INFO.clear() self.index = 0 self.build_button.text = "Run Model" - self.prev_model_settings = None + self.prev_model_settings = {} # Displays the json output from the pydantic model UI selections by user # Loops through all our stored pydantic classes @@ -1488,6 +1666,19 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): ms = now.strftime("%f")[:3] unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms + if self.pollData: + data = open_ome_zarr(self.input_directory, mode="r") + if "CurrentDimensions" in data.zattrs.keys(): + my_dict_time_indices = data.zattrs["CurrentDimensions"]["time"] + # get the prev time_index, since this is current acq + if my_dict_time_indices-1 > 1: + time_indices = list(range(0, my_dict_time_indices)) + else: + time_indices = 0 + + pollDataThread = threading.Thread(target=self.addPollLoop, args=(self.input_directory, my_dict_time_indices-1),) + pollDataThread.start() + i = 0 for item in self.pydantic_classes: i += 1 @@ -1503,7 +1694,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): # build up the arguments for the pydantic model given the current container if cls is None: - self.messageBox(ret_msg) + self.messageBox("No model defined !") return pydantic_kwargs = {} @@ -1522,20 +1713,21 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): return pydantic_kwargs["input_channel_names"] = input_channel_names - time_indices, ret_msg = self.clean_string_int_for_list( - "time_indices", pydantic_kwargs["time_indices"] - ) - if time_indices is None: - self.messageBox(ret_msg) - return - pydantic_kwargs["time_indices"] = time_indices + if not self.pollData: + time_indices, ret_msg = self.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) + if time_indices is None: + self.messageBox(ret_msg) + return + pydantic_kwargs["time_indices"] = time_indices - time_indices, ret_msg = self.clean_string_int_for_list( - "time_indices", pydantic_kwargs["time_indices"] - ) - if time_indices is None: - self.messageBox(ret_msg) - return + time_indices, ret_msg = self.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) + if time_indices is None: + self.messageBox(ret_msg) + return pydantic_kwargs["time_indices"] = time_indices if "birefringence" in pydantic_kwargs.keys(): @@ -1606,6 +1798,134 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): tableID, tableDescToolTip, proc_params ) + def addPollLoop(self, input_data_path, last_time_index): + _pydantic_classes = self.pydantic_classes.copy() + required_order = ['time', 'position', 'z', 'channel'] + _pollData = True + + tableEntryWorker = AddOTFTableEntryWorkerThread(input_data_path, True) + tableEntryWorker.add_tableOTFentry_signal.connect(self.addRemoveOTFTableEntry) + tableEntryWorker.start() + _breakFlag = False + while True: + time.sleep(10) + + try: + data = open_ome_zarr(input_data_path, mode="r") + if "CurrentDimensions" in data.zattrs.keys(): + my_dict1 = data.zattrs["CurrentDimensions"] + sorted_dict_acq = {k: my_dict1[k] for k in sorted(my_dict1, key=lambda x: required_order.index(x))} + my_dict_time_indices_curr = data.zattrs["CurrentDimensions"]["time"] + # print(sorted_dict_acq) + + if "FinalDimensions" in data.zattrs.keys(): + my_dict2 = data.zattrs["FinalDimensions"] + sorted_dict_final = {k: my_dict2[k] for k in sorted(my_dict2, key=lambda x: required_order.index(x))} + # print(sorted_dict_final) + + # use the prev time_index, since this is current acq and we need for other dims to finish acq for this t + # or when all dims match - signifying acq finished + if my_dict_time_indices_curr-2 > last_time_index or json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): + + now = datetime.datetime.now() + ms = now.strftime("%f")[:3] + unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms + + i = 0 + for item in _pydantic_classes: + i += 1 + cls = item["class"] + cls_container = item["container"] + selected_modes = item["selected_modes"] + exclude_modes = item["exclude_modes"] + c_mode_str = item["c_mode_str"] + + # gather input/out locations + input_dir = f"{item['input'].value}" + output_dir = f"{item['output'].value}" + + pydantic_kwargs = {} + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( + cls_container, cls, pydantic_kwargs, exclude_modes + ) + + input_channel_names, ret_msg = self.clean_string_for_list( + "input_channel_names", pydantic_kwargs["input_channel_names"] + ) + pydantic_kwargs["input_channel_names"] = input_channel_names + + if _pollData: + if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): + time_indices = list(range(last_time_index, my_dict_time_indices_curr)) + _breakFlag = True + else: + time_indices = list(range(last_time_index, my_dict_time_indices_curr-2)) + pydantic_kwargs["time_indices"] = time_indices + + if "birefringence" in pydantic_kwargs.keys(): + background_path, ret_msg = self.clean_path_string_when_empty( + "background_path", + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ], + ) + + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ] = background_path + + # validate and return errors if None + pydantic_model, ret_msg = self.validate_pydantic_model( + cls, pydantic_kwargs + ) + + # save the yaml files + # path is next to saved data location + save_config_path = str(Path(output_dir).parent.absolute()) + yml_file_name = "-and-".join(selected_modes) + yml_file = yml_file_name + "-" + unique_id + "-{:02d}".format(i) + ".yml" + config_path = os.path.join(save_config_path, yml_file) + utils.model_to_yaml(pydantic_model, config_path) + + expID = "{tID}-{idx}".format(tID=unique_id, idx=i) + tableID = "{tName}: ({tID}-{idx})".format( + tName=c_mode_str, tID=unique_id, idx=i + ) + tableDescToolTip = "{tName}: ({tID}-{idx})".format( + tName=yml_file_name, tID=unique_id, idx=i + ) + + proc_params = {} + proc_params["exp_id"] = expID + proc_params["desc"] = tableDescToolTip + proc_params["config_path"] = str(Path(config_path).absolute()) + proc_params["input_path"] = str(Path(input_dir).absolute()) + proc_params["output_path"] = str(Path(output_dir).absolute()) + proc_params["output_path_parent"] = str( + Path(output_dir).parent.absolute() + ) + + tableEntryWorker1 = AddTableEntryWorkerThread(tableID, tableDescToolTip, proc_params) + tableEntryWorker1.add_tableentry_signal.connect(self.addTableEntry) + tableEntryWorker1.start() + + if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final) and _breakFlag: + + tableEntryWorker2 = AddOTFTableEntryWorkerThread(input_data_path, False) + tableEntryWorker2.add_tableOTFentry_signal.connect(self.addRemoveOTFTableEntry) + tableEntryWorker2.start() + + # let child threads finish their work before exiting the parent thread + while tableEntryWorker1.isRunning() or tableEntryWorker2.isRunning(): + time.sleep(1) + time.sleep(5) + break + + last_time_index = my_dict_time_indices_curr-2 + except Exception as exc: + print(exc.args) + + # ======= These function do not implement validation # They simply make the data from GUI translate to input types # that the model expects: for eg. GUI txt field will output only str @@ -1986,86 +2306,6 @@ def _open_dialog(self, title, ref, type, filter="All Files (*)"): return path - -class CollapsibleBox(QWidget): - def __init__(self, title="", parent=None): - super(CollapsibleBox, self).__init__(parent) - - self.toggle_button = QToolButton( - text=title, checkable=True, checked=False - ) - self.toggle_button.setStyleSheet("QToolButton { border: none; }") - self.toggle_button.setToolButtonStyle( - QtCore.Qt.ToolButtonStyle.ToolButtonTextBesideIcon - ) - self.toggle_button.setArrowType(QtCore.Qt.ArrowType.RightArrow) - self.toggle_button.pressed.connect(self.on_pressed) - - self.toggle_animation = QtCore.QParallelAnimationGroup(self) - - self.content_area = QScrollArea(maximumHeight=0, minimumHeight=0) - self.content_area.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) - self.content_area.setFrameShape(QFrame.Shape.NoFrame) - - lay = QVBoxLayout(self) - lay.setSpacing(0) - lay.setContentsMargins(0, 0, 0, 0) - lay.addWidget(self.toggle_button) - lay.addWidget(self.content_area) - - self.toggle_animation.addAnimation( - QtCore.QPropertyAnimation(self, b"minimumHeight") - ) - self.toggle_animation.addAnimation( - QtCore.QPropertyAnimation(self, b"maximumHeight") - ) - self.toggle_animation.addAnimation( - QtCore.QPropertyAnimation(self.content_area, b"maximumHeight") - ) - - def setNewName(self, name): - self.toggle_button.setText(name) - - # @QtCore.pyqtSlot() - def on_pressed(self): - checked = self.toggle_button.isChecked() - self.toggle_button.setArrowType( - QtCore.Qt.ArrowType.DownArrow if not checked else QtCore.Qt.ArrowType.RightArrow - ) - self.toggle_animation.setDirection( - QtCore.QAbstractAnimation.Direction.Forward - if not checked - else QtCore.QAbstractAnimation.Direction.Backward - ) - self.toggle_animation.start() - - def setContentLayout(self, layout): - lay = self.content_area.layout() - del lay - self.content_area.setLayout(layout) - collapsed_height = ( - self.sizeHint().height() - self.content_area.maximumHeight() - ) - content_height = layout.sizeHint().height() - for i in range(self.toggle_animation.animationCount()): - animation = self.toggle_animation.animationAt(i) - animation.setDuration(500) - animation.setStartValue(collapsed_height) - animation.setEndValue(collapsed_height + content_height) - - content_animation = self.toggle_animation.animationAt( - self.toggle_animation.animationCount() - 1 - ) - content_animation.setDuration(500) - content_animation.setStartValue(0) - content_animation.setEndValue(content_height) - - -import socket, threading - - class MyWorker: def __init__(self, formLayout, tab_recon: Ui_ReconTab_Form, parentForm): @@ -2078,9 +2318,9 @@ def __init__(self, formLayout, tab_recon: Ui_ReconTab_Form, parentForm): self.threadPool = int(self.max_cores / 2) self.results = {} self.pool = None + self.futures = [] # https://click.palletsprojects.com/en/stable/testing/ - # self.runner = CliRunner() - self.startPool() + # self.runner = CliRunner() # jobs_mgmt.shared_var_jobs = self.JobsManager.shared_var_jobs self.JobsMgmt = jobs_mgmt.JobsManagement() self.JobsMgmt.clearLogs() @@ -2090,9 +2330,7 @@ def __init__(self, formLayout, tab_recon: Ui_ReconTab_Form, parentForm): thread = threading.Thread(target=self.startServer) thread.start() self.workerThreadRowDeletion = RowDeletionWorkerThread(self.formLayout) - self.workerThreadRowDeletion.removeRowSignal.connect( - self.tab_recon.removeRow - ) + self.workerThreadRowDeletion.removeRowSignal.connect(self.tab_recon.removeRow) self.workerThreadRowDeletion.start() def setNewInstances(self, formLayout, tab_recon, parentForm): @@ -2161,7 +2399,7 @@ def setPoolThreads(self, t): def startPool(self): if self.pool is None: - self.pool = ThreadPoolExecutor(max_workers=self.threadPool) + self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=self.threadPool) def shutDownPool(self): self.pool.shutdown(wait=True) @@ -2180,19 +2418,132 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s if b"\n" in buf: dataList = buf.split(b"\n") else: - dataList = [buf] + dataList = [buf] for data in dataList: if len(data)>0: decoded_string = data.decode() - json_str = str(decoded_string) - json_obj = json.loads(json_str) - for k in json_obj: - expIdx = k - jobIdx = json_obj[k]["jID"] - wellName = json_obj[k]["pos"] + if "CoNvErTeR" in decoded_string: # this request came from an agnostic route - requires processing + json_str = str(decoded_string) + json_obj = json.loads(json_str) + converter_params = json_obj["CoNvErTeR"] + input_data = converter_params["input"] + output_data = converter_params["output"] + recon_params = converter_params["params"] + expID = recon_params["expID"] + mode = recon_params["mode"] + if "config_path" in recon_params.keys(): + config_path = recon_params["config_path"] + else: + config_path = "" + + proc_params = {} + proc_params["exp_id"] = expID + proc_params["desc"] = expID + proc_params["input_path"] = str(input_data) + proc_params["output_path"] = str(output_data) + proc_params["output_path_parent"] = str(Path(output_data).parent.absolute()) + + if config_path == "": + model = None + if len(self.tab_recon.pydantic_classes) > 0: + for item in self.tab_recon.pydantic_classes: + if mode == item["selected_modes"]: + cls = item["class"] + cls_container = item["container"] + exclude_modes = item["exclude_modes"] + + # gather input/out locations + output_dir = f"{item['output'].value}" + if output_data == "": + output_data = output_dir + proc_params["output_path"] = str(output_data) + + # build up the arguments for the pydantic model given the current container + if cls is None: + self.tab_recon.messageBox("No model defined !") + return + + pydantic_kwargs = {} + pydantic_kwargs, ret_msg = self.tab_recon.get_and_validate_pydantic_args( + cls_container, cls, pydantic_kwargs, exclude_modes + ) + if pydantic_kwargs is None: + self.tab_recon.messageBox(ret_msg) + return + + input_channel_names, ret_msg = self.tab_recon.clean_string_for_list( + "input_channel_names", pydantic_kwargs["input_channel_names"] + ) + if input_channel_names is None: + self.tab_recon.messageBox(ret_msg) + return + pydantic_kwargs["input_channel_names"] = input_channel_names + + time_indices, ret_msg = self.tab_recon.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) + if time_indices is None: + self.tab_recon.messageBox(ret_msg) + return + pydantic_kwargs["time_indices"] = time_indices + + time_indices, ret_msg = self.tab_recon.clean_string_int_for_list( + "time_indices", pydantic_kwargs["time_indices"] + ) + if time_indices is None: + self.tab_recon.messageBox(ret_msg) + return + pydantic_kwargs["time_indices"] = time_indices + + if "birefringence" in pydantic_kwargs.keys(): + background_path, ret_msg = self.tab_recon.clean_path_string_when_empty( + "background_path", + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ], + ) + if background_path is None: + self.tab_recon.messageBox(ret_msg) + return + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ] = background_path + + # validate and return errors if None + pydantic_model, ret_msg = self.tab_recon.validate_pydantic_model( + cls, pydantic_kwargs + ) + if pydantic_model is None: + self.tab_recon.messageBox(ret_msg) + return + model = pydantic_model + break + if model is None: + model, msg = self.tab_recon.buildModel(mode) + yaml_path = os.path.join(str(Path(output_data).parent.absolute()), expID+".yml") + utils.model_to_yaml(model, yaml_path) + proc_params["config_path"] = str(yaml_path) + + tableEntryWorker = AddTableEntryWorkerThread(expID, expID, proc_params) + tableEntryWorker.add_tableentry_signal.connect(self.tab_recon.addTableEntry) + tableEntryWorker.start() + time.sleep(10) + return + else: + json_str = str(decoded_string) + json_obj = json.loads(json_str) + for k in json_obj: + expIdx = k + jobIdx = json_obj[k]["jID"] + wellName = json_obj[k]["pos"] + if expIdx not in self.results.keys(): # this job came from agnostic CLI route - no processing + now = datetime.datetime.now() + ms = now.strftime("%f")[:3] + unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms + expIdx = expIdx +"-"+ unique_id self.JobsMgmt.putJobInList(None, expIdx, str(jobIdx), wellName, mode="server") - thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=(expIdx, jobIdx, wellName, client_socket)) - thread.start() + thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=(expIdx, jobIdx, wellName, client_socket)) + thread.start() return except: pass @@ -2208,7 +2559,28 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s # this request came from server so we can wait for the Job to finish and update progress # some wait logic needs to be added otherwise for unknown errors this thread will persist # perhaps set a time out limit and then update the status window and then exit - params = self.results[expIdx]["JobUNK"].copy() + + if expIdx not in self.results.keys(): # this job came from agnostic route + proc_params = {} + tableID = "{exp} - {job} ({pos})".format(exp=expIdx, job=jobIdx, pos=wellName) + proc_params["exp_id"] = expIdx + proc_params["desc"] = tableID + proc_params["config_path"] = "" + proc_params["input_path"] = "" + proc_params["output_path"] = "" + proc_params["output_path_parent"] = "" + + tableEntryWorker = AddTableEntryWorkerThread(tableID, tableID, proc_params) + tableEntryWorker.add_tableentry_signal.connect(self.tab_recon.addTableEntry) + tableEntryWorker.start() + + while expIdx not in self.results.keys(): + time.sleep(1) + + params = self.results[expIdx]["JobUNK"].copy() + params["status"] = STATUS_running_job + else: + params = self.results[expIdx]["JobUNK"].copy() if jobIdx not in self.results[expIdx].keys() and len(self.results[expIdx].keys()) == 1: # this is the first job @@ -2400,19 +2772,33 @@ def clientRelease(self, expIdx, jobIdx, client_socket, params): json_str = json.dumps(json_obj)+"\n" client_socket.send(json_str.encode()) + if self.pool is not None: + print("Number of running threads:", self.pool._work_queue.qsize()) + if self.pool._work_queue.qsize() == 0: + self.pool.shutdown() + self.pool = None + def runInPool(self, params): + self.startPool() self.results[params["exp_id"]] = {} self.results[params["exp_id"]]["JobUNK"] = params self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_running_pool self.results[params["exp_id"]]["JobUNK"]["error"] = "" + try: - self.pool.submit(self.run, params) + # when a request on the listening port arrives with an empty path + # we can assume the processing was initiated outside this application + # we do not proceed with the processing and will display the results + if params["input_path"] != "": + f = self.pool.submit(self.run, params) + self.futures.append(f) except Exception as exc: self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() def runMultiInPool(self, multi_params_as_list): + self.startPool() for params in multi_params_as_list: self.results[params["exp_id"]] = {} self.results[params["exp_id"]]["JobUNK"] = params @@ -2436,11 +2822,12 @@ def getResult(self, exp_id): def run(self, params): # thread where work is passed to CLI which will handle the - # multi-processing aspects based on resources + # multi-processing aspects as Jobs if params["exp_id"] not in self.results.keys(): self.results[params["exp_id"]] = {} self.results[params["exp_id"]]["JobUNK"] = params self.results[params["exp_id"]]["JobUNK"]["error"] = "" + self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_running_pool try: # does need further threading ? probably not ! @@ -2449,17 +2836,13 @@ def run(self, params): ) thread.start() - # self.runInSubProcess(params) - - # check for this job to show up in submitit jobs list - # wait for 2 sec before raising an error - except Exception as exc: self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() def runInSubProcess(self, params): + """function that initiates the processing on the CLI""" try: input_path = str(params["input_path"]) config_path = str(params["config_path"]) @@ -2497,7 +2880,36 @@ def runInSubProcess(self, params): self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() +class AddOTFTableEntryWorkerThread(QThread): + """Worker thread for sending signal for adding component when request comes + from a different thread""" + add_tableOTFentry_signal = pyqtSignal(str, bool) + + def __init__(self, OTF_dir_path, bool_msg): + super().__init__() + self.OTF_dir_path = OTF_dir_path + self.bool_msg = bool_msg + + def run(self): + # Emit the signal to add the widget to the main thread + self.add_tableOTFentry_signal.emit(self.OTF_dir_path, self.bool_msg) +class AddTableEntryWorkerThread(QThread): + """Worker thread for sending signal for adding component when request comes + from a different thread""" + add_tableentry_signal = pyqtSignal(str, str, dict) + + def __init__(self, expID, desc, params): + super().__init__() + self.expID = expID + self.desc = desc + self.params = params + + def run(self): + # Emit the signal to add the widget to the main thread + self.add_tableentry_signal.emit(self.expID, self.desc, self.params) class AddWidgetWorkerThread(QThread): + """Worker thread for sending signal for adding component when request comes + from a different thread""" add_widget_signal = pyqtSignal(QVBoxLayout, str, str, str, str) def __init__(self, layout, expID, jID, desc, wellName): @@ -2514,9 +2926,10 @@ def run(self): ROW_POP_QUEUE = [] - -# Emits a signal to QFormLayout on the main thread class RowDeletionWorkerThread(QThread): + """Searches for a row based on its ID and then + emits a signal to QFormLayout on the main thread for deletion""" + removeRowSignal = pyqtSignal(int, str) def __init__(self, formLayout): @@ -2553,6 +2966,7 @@ def run(self): time.sleep(5) class DropButton(QPushButton): + """A drag & drop PushButton to load model file(s)""" def __init__(self, text, parent=None, recon_tab:Ui_ReconTab_Form=None): super().__init__(text, parent) self.setAcceptDrops(True) @@ -2570,6 +2984,7 @@ def dropEvent(self, event): self.recon_tab.openModelFiles(files) class DropWidget(QWidget): + """A drag & drop widget container to load model file(s) """ def __init__(self, recon_tab:Ui_ReconTab_Form=None): super().__init__() self.setAcceptDrops(True) @@ -2587,6 +3002,7 @@ def dropEvent(self, event): self.recon_tab.openModelFiles(files) class ScrollableLabel(QScrollArea): + """A scrollable label widget used for Job entry """ def __init__(self, text, *args, **kwargs): super().__init__(*args, **kwargs) @@ -2607,9 +3023,91 @@ def __init__(self, text, *args, **kwargs): def setText(self, text): self.label.setText(text) +class CollapsibleBox(QWidget): + """A collapsible widget""" + + def __init__(self, title="", parent=None, hasPydanticModel=False): + super(CollapsibleBox, self).__init__(parent) + + self.hasPydanticModel = hasPydanticModel + self.toggle_button = QToolButton( + text=title, checkable=True, checked=False + ) + self.toggle_button.setStyleSheet("QToolButton { border: none; }") + self.toggle_button.setToolButtonStyle( + QtCore.Qt.ToolButtonStyle.ToolButtonTextBesideIcon + ) + self.toggle_button.setArrowType(QtCore.Qt.ArrowType.RightArrow) + self.toggle_button.pressed.connect(self.on_pressed) + + self.toggle_animation = QtCore.QParallelAnimationGroup(self) + + self.content_area = QScrollArea(maximumHeight=0, minimumHeight=0) + self.content_area.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) + self.content_area.setFrameShape(QFrame.Shape.NoFrame) + + lay = QVBoxLayout(self) + lay.setSpacing(0) + lay.setContentsMargins(0, 0, 0, 0) + lay.addWidget(self.toggle_button) + lay.addWidget(self.content_area) + + self.toggle_animation.addAnimation( + QtCore.QPropertyAnimation(self, b"minimumHeight") + ) + self.toggle_animation.addAnimation( + QtCore.QPropertyAnimation(self, b"maximumHeight") + ) + self.toggle_animation.addAnimation( + QtCore.QPropertyAnimation(self.content_area, b"maximumHeight") + ) + + def setNewName(self, name): + self.toggle_button.setText(name) + + # @QtCore.pyqtSlot() + def on_pressed(self): + checked = self.toggle_button.isChecked() + self.toggle_button.setArrowType( + QtCore.Qt.ArrowType.DownArrow if not checked else QtCore.Qt.ArrowType.RightArrow + ) + self.toggle_animation.setDirection( + QtCore.QAbstractAnimation.Direction.Forward + if not checked + else QtCore.QAbstractAnimation.Direction.Backward + ) + self.toggle_animation.start() + if checked and self.hasPydanticModel: + # do model verification on close + pass + + def setContentLayout(self, layout): + lay = self.content_area.layout() + del lay + self.content_area.setLayout(layout) + collapsed_height = ( + self.sizeHint().height() - self.content_area.maximumHeight() + ) + content_height = layout.sizeHint().height() + for i in range(self.toggle_animation.animationCount()): + animation = self.toggle_animation.animationAt(i) + animation.setDuration(500) + animation.setStartValue(collapsed_height) + animation.setEndValue(collapsed_height + content_height) + + content_animation = self.toggle_animation.animationAt( + self.toggle_animation.animationCount() - 1 + ) + content_animation.setDuration(500) + content_animation.setStartValue(0) + content_animation.setEndValue(content_height) + + # VScode debugging if __name__ == "__main__": import napari napari.Viewer() - napari.run() + napari.run() \ No newline at end of file diff --git a/recOrder/tests/widget_tests/test_simulate_acq.py b/recOrder/tests/widget_tests/test_simulate_acq.py new file mode 100644 index 00000000..08aa63af --- /dev/null +++ b/recOrder/tests/widget_tests/test_simulate_acq.py @@ -0,0 +1,147 @@ +from pathlib import Path +from iohub.convert import TIFFConverter +from iohub.ngff import open_ome_zarr +from recOrder.cli.utils import create_empty_hcs_zarr + +import time, threading, os, shutil, json + +def convertData(tif_path, latest_out_path, prefix="", data_type_str="ometiff"): + converter = TIFFConverter( + os.path.join(tif_path , prefix), + latest_out_path, + data_type=data_type_str, + grid_layout=False, + ) + converter.run() + +def runConvert(ome_tif_path): + out_path = os.path.join(Path(ome_tif_path).parent.absolute(), ("raw_" + Path(ome_tif_path).name + ".zarr")) + convertData(ome_tif_path, out_path) + +def runAcq(input_path="", onlyPrint=False, waitBetweenT=30): + + output_store_path = os.path.join(Path(input_path).parent.absolute(), ("output_" + Path(input_path).name)) + + if Path(output_store_path).exists(): + shutil.rmtree(output_store_path) + time.sleep(1) + + input_data = open_ome_zarr(input_path, mode="r") + channel_names = input_data.channel_names + + position_keys: list[tuple[str]] = [] + + for path, pos in input_data.positions(): + # print(path) + # print(pos["0"].shape) + + shape = pos["0"].shape + dtype = pos["0"].dtype + chunks = pos["0"].chunks + scale = (1, 1, 1, 1, 1) + position_keys.append(path.split("/")) + + if onlyPrint: + print("shape: ", shape) + print("position_keys: ", position_keys) + input_data.print_tree() + return + + create_empty_hcs_zarr( + output_store_path, + position_keys, + shape, + chunks, + scale, + channel_names, + dtype, + {}, + ) + output_dataset = open_ome_zarr(output_store_path, mode="r+") + + if "Summary" in input_data.zattrs.keys(): + output_dataset.zattrs["Summary"] = input_data.zattrs["Summary"] + + output_dataset.zattrs.update({"FinalDimensions": { + "channel": shape[1], + "position": len(position_keys), + "time": shape[0], + "z": shape[2] + } + }) + + for t in range(shape[0]): + for p in range(len(position_keys)): + for z in range(shape[2]): + for c in range(shape[1]): + position_key_string = "/".join(position_keys[p]) + img_src = input_data[position_key_string][0][t, c, z] + + img_data = output_dataset[position_key_string][0] + img_data[t, c, z] = img_src + + output_dataset.zattrs.update({"CurrentDimensions": { + "channel": c+1, + "position": p+1, + "time": t+1, + "z": z+1 + } + }) + + # output_dataset.print_tree() + required_order = ['time', 'position', 'z', 'channel'] + my_dict = output_dataset.zattrs["CurrentDimensions"] + sorted_dict_acq = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} + print("Writer thread - Acquisition Dim:", sorted_dict_acq) + time.sleep(waitBetweenT) # sleep after every t + + output_dataset.close + +def runAcquire(input_path, onlyPrint, waitBetweenT): + runThread1Acq = threading.Thread(target=runAcq, args=(input_path, onlyPrint, waitBetweenT)) + runThread1Acq.start() + +def test(input_path, readerThread=True, onlyPrint=False, waitBetweenT=30): + + input_poll_data_path = os.path.join(Path(input_path).parent.absolute(), ("output_" + Path(input_path).name)) + + runAcquire(input_path, onlyPrint, waitBetweenT) + + if not readerThread: + return + + time.sleep(15) + + required_order = ['time', 'position', 'z', 'channel'] + while True: + data = open_ome_zarr(input_poll_data_path, mode="r") + print("="*60) + if "CurrentDimensions" in data.zattrs.keys(): + my_dict = data.zattrs["CurrentDimensions"] + sorted_dict_acq = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} + print("Reader thread - Acquisition Dim:", sorted_dict_acq) + + if "FinalDimensions" in data.zattrs.keys(): + my_dict = data.zattrs["FinalDimensions"] + sorted_dict_final = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} + print("Reader thread - Final Dim:", sorted_dict_final) + if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): + print("Reader thread - Acquisition Finished !") + break + print("="*60) + time.sleep(10) + +# Step 1: +# Convert an existing ome-tif recOrder acquisition, preferably with all dims (t, p, z, c) +# This will convert an existing ome-tif to a .zarr storage +# ome_tif_path = "..\\ome-zarr_data\\recOrderAcq\\test\\snap_6D_ometiff_1" +# runConvert(ome_tif_path) + +# Step 2: +# run the test to simulate Acquiring a recOrder .zarr store + +# input_path = "..\\ome-zarr_data\\recOrderAcq\\test\\raw_snap_6D_ometiff_1.zarr" +# waitBetweenT = 30 +# onlyPrint = False +# readerThread = False +# test(input_path, readerThread, onlyPrint, waitBetweenT) From e622563a95edead355cd9d7a1424f83541e54fb5 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 7 Jan 2025 02:58:15 -0500 Subject: [PATCH 17/38] Delete recOrder/tests/widget_tests/test_pydantic_model_widget.py --- .../test_pydantic_model_widget.py | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 recOrder/tests/widget_tests/test_pydantic_model_widget.py diff --git a/recOrder/tests/widget_tests/test_pydantic_model_widget.py b/recOrder/tests/widget_tests/test_pydantic_model_widget.py deleted file mode 100644 index 332e6cff..00000000 --- a/recOrder/tests/widget_tests/test_pydantic_model_widget.py +++ /dev/null @@ -1,31 +0,0 @@ -import sys -from PyQt6.QtWidgets import QApplication, QWidget, QVBoxLayout, QStyle -from PyQt6.QtGui import QPixmap -from PyQt6.QtCore import QByteArray - -from recOrder.plugin import tab_recon - -PLUGIN_NAME = "recOrder: Computational Toolkit for Label-Free Imaging" -PLUGIN_ICON = "🔬" - -class MainWindow(QWidget): - def __init__(self): - super().__init__() - recon_tab = tab_recon.Ui_ReconTab_Form(stand_alone=True) - layout = QVBoxLayout() - self.setLayout(layout) - layout.addWidget(recon_tab.recon_tab_widget) - -if __name__ == "__main__": - app = QApplication(sys.argv) - app.setStyle("Fusion") # Other options: "Fusion", "Windows", "macOS", "WindowsVista" - - window = MainWindow() - window.setWindowTitle(PLUGIN_ICON + " " + PLUGIN_NAME + " " + PLUGIN_ICON) - - pixmapi = getattr(QStyle.StandardPixmap, "SP_TitleBarMenuButton") - icon = app.style().standardIcon(pixmapi) - window.setWindowIcon(icon) - - window.show() - sys.exit(app.exec_()) \ No newline at end of file From c8d028df5010cb5bbd793cf0132571bd89a5b2a4 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 7 Jan 2025 03:01:28 -0500 Subject: [PATCH 18/38] Delete recOrder/tests/widget_tests/test_simulate_acq.py --- .../tests/widget_tests/test_simulate_acq.py | 147 ------------------ 1 file changed, 147 deletions(-) delete mode 100644 recOrder/tests/widget_tests/test_simulate_acq.py diff --git a/recOrder/tests/widget_tests/test_simulate_acq.py b/recOrder/tests/widget_tests/test_simulate_acq.py deleted file mode 100644 index 08aa63af..00000000 --- a/recOrder/tests/widget_tests/test_simulate_acq.py +++ /dev/null @@ -1,147 +0,0 @@ -from pathlib import Path -from iohub.convert import TIFFConverter -from iohub.ngff import open_ome_zarr -from recOrder.cli.utils import create_empty_hcs_zarr - -import time, threading, os, shutil, json - -def convertData(tif_path, latest_out_path, prefix="", data_type_str="ometiff"): - converter = TIFFConverter( - os.path.join(tif_path , prefix), - latest_out_path, - data_type=data_type_str, - grid_layout=False, - ) - converter.run() - -def runConvert(ome_tif_path): - out_path = os.path.join(Path(ome_tif_path).parent.absolute(), ("raw_" + Path(ome_tif_path).name + ".zarr")) - convertData(ome_tif_path, out_path) - -def runAcq(input_path="", onlyPrint=False, waitBetweenT=30): - - output_store_path = os.path.join(Path(input_path).parent.absolute(), ("output_" + Path(input_path).name)) - - if Path(output_store_path).exists(): - shutil.rmtree(output_store_path) - time.sleep(1) - - input_data = open_ome_zarr(input_path, mode="r") - channel_names = input_data.channel_names - - position_keys: list[tuple[str]] = [] - - for path, pos in input_data.positions(): - # print(path) - # print(pos["0"].shape) - - shape = pos["0"].shape - dtype = pos["0"].dtype - chunks = pos["0"].chunks - scale = (1, 1, 1, 1, 1) - position_keys.append(path.split("/")) - - if onlyPrint: - print("shape: ", shape) - print("position_keys: ", position_keys) - input_data.print_tree() - return - - create_empty_hcs_zarr( - output_store_path, - position_keys, - shape, - chunks, - scale, - channel_names, - dtype, - {}, - ) - output_dataset = open_ome_zarr(output_store_path, mode="r+") - - if "Summary" in input_data.zattrs.keys(): - output_dataset.zattrs["Summary"] = input_data.zattrs["Summary"] - - output_dataset.zattrs.update({"FinalDimensions": { - "channel": shape[1], - "position": len(position_keys), - "time": shape[0], - "z": shape[2] - } - }) - - for t in range(shape[0]): - for p in range(len(position_keys)): - for z in range(shape[2]): - for c in range(shape[1]): - position_key_string = "/".join(position_keys[p]) - img_src = input_data[position_key_string][0][t, c, z] - - img_data = output_dataset[position_key_string][0] - img_data[t, c, z] = img_src - - output_dataset.zattrs.update({"CurrentDimensions": { - "channel": c+1, - "position": p+1, - "time": t+1, - "z": z+1 - } - }) - - # output_dataset.print_tree() - required_order = ['time', 'position', 'z', 'channel'] - my_dict = output_dataset.zattrs["CurrentDimensions"] - sorted_dict_acq = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} - print("Writer thread - Acquisition Dim:", sorted_dict_acq) - time.sleep(waitBetweenT) # sleep after every t - - output_dataset.close - -def runAcquire(input_path, onlyPrint, waitBetweenT): - runThread1Acq = threading.Thread(target=runAcq, args=(input_path, onlyPrint, waitBetweenT)) - runThread1Acq.start() - -def test(input_path, readerThread=True, onlyPrint=False, waitBetweenT=30): - - input_poll_data_path = os.path.join(Path(input_path).parent.absolute(), ("output_" + Path(input_path).name)) - - runAcquire(input_path, onlyPrint, waitBetweenT) - - if not readerThread: - return - - time.sleep(15) - - required_order = ['time', 'position', 'z', 'channel'] - while True: - data = open_ome_zarr(input_poll_data_path, mode="r") - print("="*60) - if "CurrentDimensions" in data.zattrs.keys(): - my_dict = data.zattrs["CurrentDimensions"] - sorted_dict_acq = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} - print("Reader thread - Acquisition Dim:", sorted_dict_acq) - - if "FinalDimensions" in data.zattrs.keys(): - my_dict = data.zattrs["FinalDimensions"] - sorted_dict_final = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} - print("Reader thread - Final Dim:", sorted_dict_final) - if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): - print("Reader thread - Acquisition Finished !") - break - print("="*60) - time.sleep(10) - -# Step 1: -# Convert an existing ome-tif recOrder acquisition, preferably with all dims (t, p, z, c) -# This will convert an existing ome-tif to a .zarr storage -# ome_tif_path = "..\\ome-zarr_data\\recOrderAcq\\test\\snap_6D_ometiff_1" -# runConvert(ome_tif_path) - -# Step 2: -# run the test to simulate Acquiring a recOrder .zarr store - -# input_path = "..\\ome-zarr_data\\recOrderAcq\\test\\raw_snap_6D_ometiff_1.zarr" -# waitBetweenT = 30 -# onlyPrint = False -# readerThread = False -# test(input_path, readerThread, onlyPrint, waitBetweenT) From 78a0524e230b6554c43b49fc006a3061c4d78e58 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 7 Jan 2025 04:13:39 -0500 Subject: [PATCH 19/38] ditching v1.main from pydantic and reverting pydantic>=1.10.17 to test workflows --- recOrder/plugin/tab_recon.py | 14 +++++++------- setup.cfg | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index e3f835a4..b254fd3e 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -44,23 +44,23 @@ "Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format( v=version ) - ) - from pydantic.main import ModelMetaclass + ) from pydantic.main import ValidationError from pydantic.main import BaseModel - elif version >= "1.10.19": from pydantic.main import ModelMetaclass + elif version >= "1.10.19": from pydantic.main import ValidationError from pydantic.main import BaseModel + from pydantic.main import ModelMetaclass else: print( "Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format( v=version ) - ) - from pydantic.v1.main import ModelMetaclass - from pydantic.v1.main import ValidationError - from pydantic.v1.main import BaseModel + ) + from pydantic.main import ValidationError + from pydantic.main import BaseModel + from pydantic.main import ModelMetaclass except: print("Pydantic library was not found. Ver 1.10.19 is recommended.") diff --git a/setup.cfg b/setup.cfg index 9ca61f10..f36021c7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ install_requires = wget>=3.2 psutil submitit - pydantic==1.10.19 + pydantic>=1.10.17 [options.extras_require] dev = From 93bfa87f2396058b178c160c2fc9924bf8699bd8 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 7 Jan 2025 20:55:53 -0500 Subject: [PATCH 20/38] dont initialize server listening in main init of worker class - moved socket listening to sub method from init() and initialize when required for the first time (testing build & deploy) --- recOrder/cli/main.py | 2 +- recOrder/plugin/tab_recon.py | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/recOrder/cli/main.py b/recOrder/cli/main.py index b1b7a32b..2568d5ec 100644 --- a/recOrder/cli/main.py +++ b/recOrder/cli/main.py @@ -25,5 +25,5 @@ def cli(): cli.add_command(apply_inv_tf) cli.add_command(gui) -if __name__ == '__main__': +if __name__ == "__main__": cli() \ No newline at end of file diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index b254fd3e..505d40b7 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -370,8 +370,8 @@ def __init__(self, parent=None, stand_alone=False): HAS_INSTANCE["val"] = True HAS_INSTANCE["MyWorker"] = self.worker - app = QApplication.instance() - app.lastWindowClosed.connect( + self.app = QApplication.instance() + self.app.lastWindowClosed.connect( self.myCloseEvent ) # this line is connection to signal close @@ -379,6 +379,7 @@ def __init__(self, parent=None, stand_alone=False): def myCloseEvent(self): event = QEvent(QEvent.Type.Close) self.closeEvent(event) + # self.app.exit() # on napari close - cleanup def closeEvent(self, event): @@ -2323,15 +2324,20 @@ def __init__(self, formLayout, tab_recon: Ui_ReconTab_Form, parentForm): # self.runner = CliRunner() # jobs_mgmt.shared_var_jobs = self.JobsManager.shared_var_jobs self.JobsMgmt = jobs_mgmt.JobsManagement() - self.JobsMgmt.clearLogs() self.useServer = True self.serverRunning = True - self.server_socket = None - thread = threading.Thread(target=self.startServer) - thread.start() - self.workerThreadRowDeletion = RowDeletionWorkerThread(self.formLayout) - self.workerThreadRowDeletion.removeRowSignal.connect(self.tab_recon.removeRow) - self.workerThreadRowDeletion.start() + self.server_socket = None + self.isInitialized = False + + def initialize(self): + if not self.isInitialized: + thread = threading.Thread(target=self.startServer) + thread.start() + self.workerThreadRowDeletion = RowDeletionWorkerThread(self.formLayout) + self.workerThreadRowDeletion.removeRowSignal.connect(self.tab_recon.removeRow) + self.workerThreadRowDeletion.start() + self.JobsMgmt.clearLogs() + self.isInitialized = True def setNewInstances(self, formLayout, tab_recon, parentForm): self.formLayout: QFormLayout = formLayout @@ -2779,6 +2785,9 @@ def clientRelease(self, expIdx, jobIdx, client_socket, params): self.pool = None def runInPool(self, params): + if not self.isInitialized: + self.initialize() + self.startPool() self.results[params["exp_id"]] = {} self.results[params["exp_id"]]["JobUNK"] = params From 086b5ff3e0364e09fa9a1439c651b4fb194a63e9 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Thu, 9 Jan 2025 02:04:42 -0500 Subject: [PATCH 21/38] incorporating discussed GUI changes - GUI changes to reflect sketch as discussed with @talonchandler @ieivanov - Output directory and validation - Open dataset after reconstruction based on user choice - pinning pydantic lib back to 1.10.19 --- recOrder/cli/gui_widget.py | 6 +- recOrder/plugin/gui.py | 4 +- recOrder/plugin/main_widget.py | 1 + recOrder/plugin/tab_recon.py | 778 ++++++++++++++++++--------------- setup.cfg | 2 +- 5 files changed, 440 insertions(+), 351 deletions(-) diff --git a/recOrder/cli/gui_widget.py b/recOrder/cli/gui_widget.py index 14f68872..5d2b2873 100644 --- a/recOrder/cli/gui_widget.py +++ b/recOrder/cli/gui_widget.py @@ -17,7 +17,7 @@ def gui(): app = QApplication(sys.argv) app.setStyle("Fusion") # Other options: "Fusion", "Windows", "macOS", "WindowsVista" try: - qdarktheme.setup_theme('auto') + qdarktheme.setup_theme("dark") except:pass window = MainWindow() window.setWindowTitle(PLUGIN_ICON + " " + PLUGIN_NAME + " " + PLUGIN_ICON) @@ -35,7 +35,7 @@ def __init__(self): recon_tab = tab_recon.Ui_ReconTab_Form(stand_alone=True) layout = QVBoxLayout() self.setLayout(layout) - layout.addWidget(recon_tab.recon_tab_widget) + layout.addWidget(recon_tab.recon_tab_mainScrollArea) -if __name__ == '__main__': +if __name__ == "__main__": gui() \ No newline at end of file diff --git a/recOrder/plugin/gui.py b/recOrder/plugin/gui.py index f9e53cb0..0e27c00b 100644 --- a/recOrder/plugin/gui.py +++ b/recOrder/plugin/gui.py @@ -925,8 +925,8 @@ def setupUi(self, Form): self.gridLayout_6.addWidget(self.scrollArea_4, 4, 0, 1, 1) self.tabWidget.addTab(self.Acquisition, "") - self.recon_tab = tab_recon.Ui_ReconTab_Form(Form) - self.tabWidget.addTab(self.recon_tab.recon_tab_mainScrollArea, 'Reconstruction') + self.tab_reconstruction = tab_recon.Ui_ReconTab_Form(Form) + self.tabWidget.addTab(self.tab_reconstruction.recon_tab_mainScrollArea, 'Reconstruction') self.Display = QtWidgets.QWidget() self.Display.setObjectName("Display") diff --git a/recOrder/plugin/main_widget.py b/recOrder/plugin/main_widget.py index d27b52a6..947ac537 100644 --- a/recOrder/plugin/main_widget.py +++ b/recOrder/plugin/main_widget.py @@ -90,6 +90,7 @@ def __init__(self, napari_viewer: Viewer): # Setup GUI elements self.ui = gui.Ui_Form() self.ui.setupUi(self) + self.ui.tab_reconstruction.setViewer(napari_viewer) # Override initial tab focus self.ui.tabWidget.setCurrentIndex(0) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 505d40b7..9b51f201 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1,4 +1,4 @@ -import os, json, subprocess, time, datetime, uuid +import sys, os, json, subprocess, time, datetime, uuid import socket, threading from pathlib import Path @@ -15,6 +15,8 @@ from magicgui.type_map import get_widget_class import warnings +from napari import Viewer + from recOrder.io import utils from recOrder.cli import settings, jobs_mgmt from napari.utils import notifications @@ -114,99 +116,103 @@ def __init__(self, parent=None, stand_alone=False): super().__init__(parent) self._ui = parent self.stand_alone = stand_alone + self.viewer:Viewer = None if HAS_INSTANCE["val"]: self.current_dir_path = str(Path.cwd()) self.directory = str(Path.cwd()) - self.current_save_path = HAS_INSTANCE["current_save_path"] self.input_directory = HAS_INSTANCE["input_directory"] - self.save_directory = HAS_INSTANCE["save_directory"] + self.output_directory = HAS_INSTANCE["output_directory"] self.model_directory = HAS_INSTANCE["model_directory"] self.yaml_model_file = HAS_INSTANCE["yaml_model_file"] else: self.directory = str(Path.cwd()) self.current_dir_path = str(Path.cwd()) - self.current_save_path = str(Path.cwd()) self.input_directory = str(Path.cwd()) - self.save_directory = str(Path.cwd()) + self.output_directory = str(Path.cwd()) self.model_directory = str(Path.cwd()) self.yaml_model_file = str(Path.cwd()) self.input_directory_dataset = None self.input_directory_datasetMeta = None - # Top level parent + # Parent (Widget) which holds the GUI ############################## + self.recon_tab_mainScrollArea = QScrollArea() + self.recon_tab_mainScrollArea.setWidgetResizable(True) + self.recon_tab_widget = QWidget() + self.recon_tab_widget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) self.recon_tab_layout = QVBoxLayout() self.recon_tab_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) self.recon_tab_layout.setContentsMargins(0, 0, 0, 0) self.recon_tab_layout.setSpacing(0) self.recon_tab_widget.setLayout(self.recon_tab_layout) - - self.recon_tab_mainScrollArea = QScrollArea() - self.recon_tab_mainScrollArea.setWidgetResizable(True) self.recon_tab_mainScrollArea.setWidget(self.recon_tab_widget) - # Top level - Data Input - self.modes_widget2 = QWidget() - self.modes_layout2 = QHBoxLayout() - self.modes_layout2.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - self.modes_widget2.setLayout(self.modes_layout2) - self.modes_widget2.setMaximumHeight(50) - self.modes_widget2.setMinimumHeight(50) + # Top Section Group - Data ############################## + group_box_Data_groupBox_widget = QGroupBox("Data") + group_box_Data_layout = QVBoxLayout() + group_box_Data_layout.setContentsMargins(0, 5, 0, 0) + group_box_Data_layout.setSpacing(0) + group_box_Data_groupBox_widget.setLayout(group_box_Data_layout) + + # Input Data ############################## + self.data_input_widget = QWidget() + self.data_input_widget_layout = QHBoxLayout() + self.data_input_widget_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.data_input_widget.setLayout(self.data_input_widget_layout) + + self.data_input_Label = widgets.Label(value="Input Store") + # self.data_input_Label.native.setMinimumWidth(97) + self.data_input_LineEdit = widgets.LineEdit(value=self.input_directory) + self.data_input_PushButton = widgets.PushButton(label="Browse") + # self.data_input_PushButton.native.setMinimumWidth(75) + self.data_input_PushButton.clicked.connect(self.browse_dir_path_input) + self.data_input_LineEdit.changed.connect(self.readAndSetInputPathOnValidation) + + self.data_input_widget_layout.addWidget(self.data_input_Label.native) + self.data_input_widget_layout.addWidget(self.data_input_LineEdit.native) + self.data_input_widget_layout.addWidget(self.data_input_PushButton.native) + + # Output Data ############################## + self.data_output_widget = QWidget() + self.data_output_widget_layout = QHBoxLayout() + self.data_output_widget_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.data_output_widget.setLayout(self.data_output_widget_layout) + + self.data_output_Label = widgets.Label(value="Output Directory") + self.data_output_LineEdit = widgets.LineEdit(value=self.output_directory) + self.data_output_PushButton = widgets.PushButton(label="Browse") + # self.data_output_PushButton.native.setMinimumWidth(75) + self.data_output_PushButton.clicked.connect(self.browse_dir_path_output) + self.data_output_LineEdit.changed.connect(self.readAndSetOutPathOnValidation) + + self.data_output_widget_layout.addWidget(self.data_output_Label.native) + self.data_output_widget_layout.addWidget(self.data_output_LineEdit.native) + self.data_output_widget_layout.addWidget(self.data_output_PushButton.native) - self.reconstruction_input_data_label = widgets.Label( - name="", value="Input Data" - ) - self.reconstruction_input_data_loc = widgets.LineEdit( - name="", value=self.input_directory - ) - self.reconstruction_input_data_btn = widgets.PushButton( - name="InputData", label="Browse" - ) - self.reconstruction_input_data_btn.native.setMinimumWidth(75) - self.reconstruction_input_data_btn.clicked.connect( - self.browse_dir_path_input - ) - self.reconstruction_input_data_loc.changed.connect( - self.readAndSetInputPathOnValidation - ) - _load_model_btn = DropButton(text="Load Model(s)", recon_tab=self) - _load_model_btn.setMinimumWidth(90) + self.data_input_Label.native.setMinimumWidth(115) + self.data_output_Label.native.setMinimumWidth(115) - self.modes_layout2.addWidget(self.reconstruction_input_data_label.native) - self.modes_layout2.addWidget(self.reconstruction_input_data_loc.native) - self.modes_layout2.addWidget(self.reconstruction_input_data_btn.native) - self.modes_layout2.addWidget(_load_model_btn) - self.recon_tab_layout.addWidget(self.modes_widget2) + group_box_Data_layout.addWidget(self.data_input_widget) + group_box_Data_layout.addWidget(self.data_output_widget) + self.recon_tab_layout.addWidget(group_box_Data_groupBox_widget) - # _load_model_label = widgets.Label(name="", value="Models Path") - # _load_model_loc = widgets.LineEdit(name="", value=self.model_directory) - - # Passing model location label to model location selector - _load_model_btn.clicked.connect( - lambda: self.browse_dir_path_model() - ) - - # HBox for Loading Model - # _hBox_widget_model = QWidget() - # _hBox_layout_model = QHBoxLayout() - # _hBox_layout_model.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - # _hBox_widget_model.setLayout(_hBox_layout_model) - # _hBox_widget_model.setMaximumHeight(50) - # _hBox_widget_model.setMinimumHeight(50) - # _hBox_layout_model.addWidget(_load_model_label.native) - # _hBox_layout_model.addWidget(_load_model_loc.native) - # _hBox_layout_model.addWidget(_load_model_btn) - - # self.recon_tab_layout.addWidget(_hBox_widget_model) + ################################## + + # Middle Section - Models ############################## + # Selection modes, New, Load, Clear + # Pydantic Models ScrollArea + + group_box_Models_groupBox_widget = QGroupBox("Models") + group_box_Models_layout = QVBoxLayout() + group_box_Models_layout.setContentsMargins(0, 5, 0, 0) + group_box_Models_layout.setSpacing(0) + group_box_Models_groupBox_widget.setLayout(group_box_Models_layout) - # Top level - Selection modes, model creation and running - self.modes_widget = QWidget() - self.modes_layout = QHBoxLayout() - self.modes_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - self.modes_widget.setLayout(self.modes_layout) - self.modes_widget.setMaximumHeight(50) - self.modes_widget.setMinimumHeight(50) + self.models_widget = QWidget() + self.models_widget_layout = QHBoxLayout() + self.models_widget_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.models_widget.setLayout(self.models_widget_layout) self.modes_selected = OPTION_TO_MODEL_DICT.copy() @@ -219,146 +225,129 @@ def __init__(self, parent=None, stand_alone=False): self.modes_selected[mode]["Checkbox"] = widgets.Checkbox( name=mode, label=mode ) - self.modes_layout.addWidget( + self.models_widget_layout.addWidget( self.modes_selected[mode]["Checkbox"].native ) # PushButton to create a copy of the model - UI - self.reconstruction_mode_enabler = widgets.PushButton( - name="BuildModel", label="Build Model" + self.models_new_PushButton = widgets.PushButton( + label="New" ) - self.reconstruction_mode_enabler.native.setMinimumWidth(100) - self.reconstruction_mode_enabler.clicked.connect( + # self.models_new_PushButton.native.setMinimumWidth(100) + self.models_new_PushButton.clicked.connect( self._build_acq_contols ) + self.models_load_PushButton = DropButton(text="Load", recon_tab=self) + # self.models_load_PushButton.setMinimumWidth(90) + + # Passing model location label to model location selector + self.models_load_PushButton.clicked.connect( + lambda: self.browse_dir_path_model()) + # PushButton to clear all copies of models that are create for UI - self.reconstruction_mode_clear = widgets.PushButton( - name="ClearModels", label="Clear All Models" + self.models_clear_PushButton = widgets.PushButton( + label="Clear" ) - self.reconstruction_mode_clear.native.setMinimumWidth(110) - self.reconstruction_mode_clear.clicked.connect(self._clear_all_models) - - # PushButton to validate and create the yaml file(s) based on selection - self.build_button = widgets.PushButton(name="Run Model") - self.build_button.native.setMinimumWidth(100) - self.build_button.clicked.connect(self.build_model_and_run) - - # Editable List holding pydantic class(es) as per user selection - self.pydantic_classes = list() - self.prev_model_settings = {} - self.index = 0 - self.pollData = False + # self.models_clear_PushButton.native.setMinimumWidth(110) + self.models_clear_PushButton.clicked.connect(self._clear_all_models) - self.modes_layout.addWidget(self.reconstruction_mode_enabler.native) - self.modes_layout.addWidget(self.reconstruction_mode_clear.native) - self.modes_layout.addWidget(self.build_button.native) - self.recon_tab_layout.addWidget(self.modes_widget) + self.models_widget_layout.addWidget(self.models_new_PushButton.native) + self.models_widget_layout.addWidget(self.models_load_PushButton) + self.models_widget_layout.addWidget(self.models_clear_PushButton.native) + + # Middle scrollable component which will hold Editable/(vertical) Expanding UI + self.models_scrollArea = QScrollArea() + self.models_scrollArea.setWidgetResizable(True) + self.models_container_widget = DropWidget(self) + self.models_container_widget_layout = QVBoxLayout() + self.models_container_widget_layout.setContentsMargins(0, 0, 0, 0) + self.models_container_widget_layout.setSpacing(2) + self.models_container_widget_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.models_container_widget.setLayout(self.models_container_widget_layout) + self.models_scrollArea.setWidget(self.models_container_widget) - # Line seperator between top / middle UI components - _line = QFrame() - _line.setMinimumWidth(1) - _line.setFixedHeight(2) - _line.setFrameShape(QFrame.HLine) - _line.setFrameShadow(QFrame.Sunken) - _line.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) - _line.setStyleSheet( - "margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;" - ) - self.recon_tab_layout.addWidget(_line) - - # Top level - Central scrollable component which will hold Editable/(vertical) Expanding UI - group_box_process_models = QGroupBox("Processing Models") - group_box_process_models.setMinimumHeight(200) - group_box_process_models_layout = QHBoxLayout() - group_box_process_models.setLayout(group_box_process_models_layout) - self.recon_tab_scrollArea_settings = QScrollArea() - # self.recon_tab_scrollArea_settings.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) - self.recon_tab_scrollArea_settings.setWidgetResizable(True) - self.recon_tab_qwidget_settings = DropWidget(self) - self.recon_tab_qwidget_settings_layout = QVBoxLayout() - self.recon_tab_qwidget_settings_layout.setSpacing(10) - self.recon_tab_qwidget_settings_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - self.recon_tab_qwidget_settings.setLayout( - self.recon_tab_qwidget_settings_layout - ) - self.recon_tab_scrollArea_settings.setWidget( - self.recon_tab_qwidget_settings - ) + group_box_Models_layout.addWidget(self.models_widget) + group_box_Models_layout.addWidget(self.models_scrollArea) + + ################################## - # Create the splitter + # Create the splitter to resize Middle and Bottom Sections if required ################################## splitter = QSplitter() splitter.setOrientation(Qt.Orientation.Vertical) splitter.setSizes([600, 200]) + self.recon_tab_layout.addWidget(splitter) - - group_box_process_models_layout.addWidget(self.recon_tab_scrollArea_settings) - splitter.addWidget(group_box_process_models) - - _scrollArea = QScrollArea() - _scrollArea.setWidgetResizable(True) - _qwidget_settings = QWidget() - _qwidget_settings.setSizePolicy( - QSizePolicy.Expanding, QSizePolicy.Expanding - ) - # _qwidget_settings.setStyleSheet( - # "margin:1px; padding:2px; border:1px solid rgb(128,128,128); border-width: 1px;" - # ) - _qwidget_settings_layout = QVBoxLayout() - _qwidget_settings_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - _qwidget_settings.setLayout(_qwidget_settings_layout) - _scrollArea.setWidget(_qwidget_settings) - splitter.addWidget(_scrollArea) + + # Reconstruction ################################## + # Run, Processing, On-The-Fly + group_box_Reconstruction_groupBox_widget = QGroupBox("Reconstruction Queue") + group_box_Reconstruction_layout = QVBoxLayout() + group_box_Reconstruction_layout.setContentsMargins(5, 10, 5, 5) + group_box_Reconstruction_layout.setSpacing(2) + group_box_Reconstruction_groupBox_widget.setLayout(group_box_Reconstruction_layout) + + splitter.addWidget(group_box_Models_groupBox_widget) + splitter.addWidget(group_box_Reconstruction_groupBox_widget) my_splitter_handle = splitter.handle(1) my_splitter_handle.setStyleSheet("background: 1px rgb(128,128,128);") splitter.setStyleSheet("""QSplitter::handle:pressed {background-color: #ca5;}""") - - # Table for processing entries - group_box_OTF = QGroupBox("On-The-Fly Processing Queue") - group_box_OTF_layout = QHBoxLayout() - group_box_OTF.setLayout(group_box_OTF_layout) + # PushButton to validate and Run the yaml file(s) based on selection against the Input store + self.reconstruction_run_PushButton = widgets.PushButton(name="RUN Model") + self.reconstruction_run_PushButton.native.setMinimumWidth(100) + self.reconstruction_run_PushButton.clicked.connect(self.build_model_and_run) + + group_box_Reconstruction_layout.addWidget(self.reconstruction_run_PushButton.native) + + # Tabs - Processing & On-The-Fly + tabs_Reconstruction = QTabWidget() + group_box_Reconstruction_layout.addWidget(tabs_Reconstruction) + + # Table for Jobs processing entries + tab1_processing_widget = QWidget() + tab1_processing_widget_layout = QVBoxLayout() + tab1_processing_widget_layout.setContentsMargins(5, 5, 5, 5) + tab1_processing_widget_layout.setSpacing(2) + tab1_processing_widget.setLayout(tab1_processing_widget_layout) + self.proc_table_QFormLayout = QFormLayout() + self.proc_table_QFormLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + tab1_processing_form_widget = QWidget() + tab1_processing_form_widget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + tab1_processing_form_widget.setLayout(self.proc_table_QFormLayout) + tab1_processing_widget_layout.addWidget(tab1_processing_form_widget) + + _clear_results_btn = widgets.PushButton(label="Clear Results") + _clear_results_btn.clicked.connect(self.clear_results_table) + tab1_processing_widget_layout.addWidget(_clear_results_btn.native) + + # Table for On-The-Fly processing entries + tab2_processing_widget = QWidget() + tab2_processing_widget_layout = QVBoxLayout() + tab2_processing_widget_layout.setContentsMargins(0, 0, 0, 0) + tab2_processing_widget_layout.setSpacing(0) + tab2_processing_widget.setLayout(tab2_processing_widget_layout) self.proc_OTF_table_QFormLayout = QFormLayout() self.proc_OTF_table_QFormLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - self.proc_OTF_table_QFormLayout.setSpacing(0) - self.proc_OTF_table_QFormLayout.setContentsMargins(0, 0, 0, 0) _proc_OTF_table_widget = QWidget() - _proc_OTF_table_widget.setSizePolicy( - QSizePolicy.Expanding, QSizePolicy.Expanding - ) + _proc_OTF_table_widget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) _proc_OTF_table_widget.setLayout(self.proc_OTF_table_QFormLayout) - group_box_OTF_layout.addWidget(_proc_OTF_table_widget) - group_box_OTF.setMaximumHeight(100) + tab2_processing_widget_layout.addWidget(_proc_OTF_table_widget) + tab2_processing_widget.setMaximumHeight(100) - group_box_JobResults = QGroupBox("Job Results Processing Queue") - group_box_JobResults_layout = QHBoxLayout() - group_box_JobResults.setLayout(group_box_JobResults_layout) - self.proc_table_QFormLayout = QFormLayout() - self.proc_table_QFormLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - self.proc_table_QFormLayout.setSpacing(0) - self.proc_table_QFormLayout.setContentsMargins(0, 0, 0, 0) - _proc_table_widget = QWidget() - _proc_table_widget.setSizePolicy( - QSizePolicy.Expanding, QSizePolicy.Expanding - ) - _proc_table_widget.setLayout(self.proc_table_QFormLayout) - group_box_JobResults_layout.addWidget(_proc_table_widget) + tabs_Reconstruction.addTab(tab1_processing_widget, "Processing") + tabs_Reconstruction.addTab(tab2_processing_widget, "On-The-Fly") - _clear_results_btn = widgets.PushButton( - name="ClearResults", label="Clear Results" - ) - _clear_results_btn.clicked.connect(self.clear_results_table) - _qwidget_settings_layout.addWidget(group_box_OTF) - _qwidget_settings_layout.addWidget(_clear_results_btn.native) - _qwidget_settings_layout.addWidget(group_box_JobResults) + # Editable List holding pydantic class(es) as per user selection + self.pydantic_classes = list() + self.prev_model_settings = {} + self.index = 0 + self.pollData = False # Stores Model & Components values which cause validation failure - can be highlighted on the model field as Red self.modelHighlighterVals = {} - # Flag to delete Process update table row on successful Job completion - # self.autoDeleteRowOnCompletion = True - # handle napari's close widget and avoid starting a second server if HAS_INSTANCE["val"]: self.worker: MyWorker = HAS_INSTANCE["MyWorker"] @@ -375,6 +364,8 @@ def __init__(self, parent=None, stand_alone=False): self.myCloseEvent ) # this line is connection to signal close + ###################################################### + # our defined close event since napari doesnt do def myCloseEvent(self): event = QEvent(QEvent.Type.Close) @@ -396,6 +387,17 @@ def showEvent(self, event): if event.type() == QEvent.Type.Show: pass + def setViewer(self, viewer): + self.viewer = viewer + + def showDataset(self, data_path): + # Show reconstruction data + try: + if self.viewer is not None: + self.viewer.open(data_path, plugin="napari-ome-zarr") + except Exception as exc: + self.messageBox(exc.args) + def confirmDialog(self, msg="Confirm your selection ?"): qm = QMessageBox ret = qm.question( @@ -418,15 +420,32 @@ def browse_dir_path_input(self): else: self._clear_all_models(silent=True) try: - result = self._open_file_dialog(self.input_directory, "dir") + result = self._open_file_dialog(self.input_directory, "dir", filter="ZARR Storage (*.zarr)") + # .zarr is a folder but we could implement a filter to scan for "ending with" and present those if required + except Exception as exc: + self.messageBox(exc.args) + return + + if result == "": + return + + self.data_input_LineEdit.value = result + + def browse_dir_path_output(self): + try: + result = self._open_file_dialog(self.output_directory, "dir") except Exception as exc: self.messageBox(exc.args) return if result == "": return + + if not Path(result).exists(): + self.messageBox("Output Directory path must exist !") + return - self.reconstruction_input_data_loc.value = result + self.data_output_LineEdit.value = result def browse_dir_path_inputBG(self, elem): result = self._open_file_dialog(self.directory, "dir") @@ -472,18 +491,18 @@ def validateInputData( # include data validation def readAndSetInputPathOnValidation(self): if ( - self.reconstruction_input_data_loc.value is None - or len(self.reconstruction_input_data_loc.value) == 0 + self.data_input_LineEdit.value is None + or len(self.data_input_LineEdit.value) == 0 ): - self.reconstruction_input_data_loc.value = self.input_directory + self.data_input_LineEdit.value = self.input_directory self.messageBox("Input data path cannot be empty") return - if not Path(self.reconstruction_input_data_loc.value).exists(): - self.reconstruction_input_data_loc.value = self.input_directory + if not Path(self.data_input_LineEdit.value).exists(): + self.data_input_LineEdit.value = self.input_directory self.messageBox("Input data path must point to a valid location") return - result = self.reconstruction_input_data_loc.value + result = self.data_input_LineEdit.value valid, ret_msg = self.validateInputData(result) if valid: @@ -495,9 +514,43 @@ def readAndSetInputPathOnValidation(self): self.saveLastPaths() else: - self.reconstruction_input_data_loc.value = self.input_directory + self.data_input_LineEdit.value = self.input_directory self.messageBox(ret_msg) + self.data_output_LineEdit.value = Path(self.input_directory).parent.absolute() + + def readAndSetOutPathOnValidation(self): + if ( + self.data_output_LineEdit.value is None + or len(self.data_output_LineEdit.value) == 0 + ): + self.data_output_LineEdit.value = self.output_directory + self.messageBox("Output data path cannot be empty") + return + if not Path(self.data_output_LineEdit.value).exists(): + self.data_output_LineEdit.value = self.output_directory + self.messageBox("Output data path must point to a valid location") + return + + self.output_directory = self.data_output_LineEdit.value + + self.validateModelOutputPaths() + + def validateModelOutputPaths(self): + if len(self.pydantic_classes) > 0: + for model_item in self.pydantic_classes: + output_LineEdit = model_item["output_LineEdit"] + output_Button = model_item["output_Button"] + + full_out_path = os.path.join(Path(self.output_directory).absolute(), output_LineEdit.value) + model_item["output"] = full_out_path + + save_path_exists = True if Path(full_out_path).exists() else False + output_LineEdit.label = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:" + output_LineEdit.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") + output_Button.text = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:" + output_Button.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") + def isDatasetAcqRunning(self, zattrs: dict)->bool: """ Checks the zattrs for CurrentDimensions & FinalDimensions key and tries to figure if @@ -518,32 +571,30 @@ def isDatasetAcqRunning(self, zattrs: dict)->bool: # Copied from main_widget # ToDo: utilize common functions # Output data selector - def browse_dir_path_output(self, elem): - result = self._open_file_dialog(self.save_directory, "save") + def browse_model_dir_path_output(self, elem): + result = self._open_file_dialog(self.output_directory, "save") if result == "": return save_path_exists = True if Path(result).exists() else False - elem.label = "Output Data" + ("" if not save_path_exists else (" "+_validate_alert)) + elem.label = "Output Data:" + ("" if not save_path_exists else (" "+_validate_alert)) elem.tooltip ="" if not save_path_exists else "Output file exists" - - self.directory = result - self.save_directory = result - elem.value = self.save_directory + + elem.value = Path(result).name self.saveLastPaths() # call back for output LineEdit path changed manually def readAndSetOutputPathOnValidation(self, elem1, elem2, save_path): if elem1.value is None or len(elem1.value) == 0: - elem1.value = save_path + elem1.value = Path(save_path).name - save_path = elem1.value + save_path = os.path.join(Path(self.output_directory).absolute(), elem1.value) save_path_exists = True if Path(save_path).exists() else False - elem1.label = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data" + elem1.label = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:" elem1.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") - elem2.text = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data" + elem2.text = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:" elem2.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") self.saveLastPaths() @@ -657,9 +708,8 @@ def openModelFiles(self, results:List): # useful when using close widget and not napari close and we might need them again def saveLastPaths(self): HAS_INSTANCE["current_dir_path"] = self.current_dir_path - HAS_INSTANCE["current_save_path"] = self.current_save_path HAS_INSTANCE["input_directory"] = self.input_directory - HAS_INSTANCE["save_directory"] = self.save_directory + HAS_INSTANCE["output_directory"] = self.output_directory HAS_INSTANCE["model_directory"] = self.model_directory HAS_INSTANCE["yaml_model_file"] = self.yaml_model_file @@ -922,12 +972,10 @@ def add_widget(self, parentLayout:QVBoxLayout, expID, jID, tableEntryID="", pos= ) _scrollAreaCollapsibleBox = QScrollArea() _scrollAreaCollapsibleBox.setWidgetResizable(True) - _scrollAreaCollapsibleBox.setMinimumHeight(600) + _scrollAreaCollapsibleBox.setMinimumHeight(300) _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) _collapsibleBoxWidgetLayout = QVBoxLayout() - _collapsibleBoxWidgetLayout.setContentsMargins(0, 0, 0, 0) - _collapsibleBoxWidgetLayout.setSpacing(0) _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) _collapsibleBoxWidget = CollapsibleBox( @@ -1017,14 +1065,12 @@ def addTableEntry( _scrollAreaCollapsibleBox = QScrollArea() _scrollAreaCollapsibleBox.setWidgetResizable(True) _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) - _scrollAreaCollapsibleBox.setMinimumHeight(600) + _scrollAreaCollapsibleBox.setMinimumHeight(300) _scrollAreaCollapsibleBox.setSizePolicy( QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed ) _collapsibleBoxWidgetLayout = QVBoxLayout() - _collapsibleBoxWidgetLayout.setContentsMargins(0, 0, 0, 0) - _collapsibleBoxWidgetLayout.setSpacing(0) _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) _collapsibleBoxWidget = CollapsibleBox(tableEntryID) @@ -1040,8 +1086,6 @@ def addTableEntry( _expandingTabEntryWidget = QWidget() _expandingTabEntryWidget.toolTip = tableEntryShortDesc _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.layout().setContentsMargins(0, 0, 0, 0) - _expandingTabEntryWidget.layout().setSpacing(0) _expandingTabEntryWidget.setSizePolicy( QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed ) @@ -1270,18 +1314,18 @@ def _create_acq_contols2( # These could be multiple based on user selection for each model # Inherits from Input by default at creation time name_without_ext = os.path.splitext(self.input_directory)[0] - save_path = os.path.join(Path(self.input_directory).parent.absolute(), (name_without_ext + ("_"+c_mode_short+"_"+num_str) + ".zarr")) + save_path = os.path.join(Path(self.output_directory).parent.absolute(), (name_without_ext + ("_"+c_mode_short+"_"+num_str) + ".zarr")) save_path_exists = True if Path(save_path).exists() else False _output_data_loc = widgets.LineEdit( - name="", value=save_path, tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") + value=Path(save_path).name, tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") ) _output_data_btn = widgets.PushButton( - name= "OutputData", text= ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data", tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") + text= ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:", tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") ) # Passing location label to output location selector _output_data_btn.clicked.connect( - lambda: self.browse_dir_path_output(_output_data_loc) + lambda: self.browse_model_dir_path_output(_output_data_loc) ) _output_data_loc.changed.connect( lambda: self.readAndSetOutputPathOnValidation(_output_data_loc, _output_data_btn, save_path) @@ -1306,9 +1350,9 @@ def _create_acq_contols2( _hBox_widget = QWidget() _hBox_layout = QHBoxLayout() _hBox_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - _hBox_widget.setLayout(_hBox_layout) - _hBox_layout.addWidget(_output_data_loc.native) + _hBox_widget.setLayout(_hBox_layout) _hBox_layout.addWidget(_output_data_btn.native) + _hBox_layout.addWidget(_output_data_loc.native) # Add this container to the main scrollable widget _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() @@ -1324,21 +1368,21 @@ def _create_acq_contols2( _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) _collapsibleBoxWidgetLayout = QVBoxLayout() - _collapsibleBoxWidgetLayout.setContentsMargins(0, 0, 0, 0) - _collapsibleBoxWidgetLayout.setSpacing(0) _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) _collapsibleBoxWidget = CollapsibleBox( c_mode_str ) # tableEntryID, tableEntryShortDesc - should update with processing status - - _validate_button = widgets.PushButton(name="Validate Model") + + _show_CheckBox = widgets.CheckBox(name="Show after Reconstruction", value=True) + _validate_button = widgets.PushButton(name="Validate") _validate_button.clicked.connect(lambda:self._validate_model(_str, _collapsibleBoxWidget)) _hBox_widget2 = QWidget() _hBox_layout2 = QHBoxLayout() _hBox_layout2.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _hBox_widget2.setLayout(_hBox_layout2) + _hBox_layout2.addWidget(_show_CheckBox.native) _hBox_layout2.addWidget(_validate_button.native) _hBox_layout2.addWidget(_del_button.native) @@ -1347,8 +1391,6 @@ def _create_acq_contols2( _expandingTabEntryWidget.toolTip = c_mode_str _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.layout().setContentsMargins(0, 0, 0, 0) - _expandingTabEntryWidget.layout().setSpacing(0) _expandingTabEntryWidget.setSizePolicy( QSizePolicy.Expanding, QSizePolicy.Fixed ) @@ -1367,7 +1409,7 @@ def _create_acq_contols2( ) _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) - self.recon_tab_qwidget_settings_layout.addWidget( + self.models_container_widget_layout.addWidget( _expandingTabEntryWidget ) @@ -1381,22 +1423,25 @@ def _create_acq_contols2( "c_mode_str": c_mode_str, "collapsibleBoxWidget": _collapsibleBoxWidget, "class": pydantic_class, - "input": self.reconstruction_input_data_loc, - "output": _output_data_loc, + "input": self.data_input_LineEdit, + "output": os.path.join(Path(self.output_directory).absolute(), _output_data_loc.value), + "output_LineEdit": _output_data_loc, + "output_Button": _output_data_btn, "container": recon_pydantic_container, "selected_modes": selected_modes.copy(), "exclude_modes": exclude_modes.copy(), "poll_data": self.pollData, + "show": _show_CheckBox.value, } ) self.index += 1 if self.index > 1: - self.build_button.text = "Run {n} Models".format( + self.reconstruction_run_PushButton.text = "RUN {n} Models".format( n=self.index ) else: - self.build_button.text = "Run Model" + self.reconstruction_run_PushButton.text = "RUN Model" return pydantic_model @@ -1528,19 +1573,19 @@ def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, index, _str): i += 1 self.index = len(self.pydantic_classes) if self.index > 1: - self.build_button.text = "Run {n} Models".format( + self.reconstruction_run_PushButton.text = "RUN {n} Models".format( n=self.index ) else: - self.build_button.text = "Run Model" + self.reconstruction_run_PushButton.text = "RUN Model" # Clear all the generated pydantic models and clears the pydantic model list def _clear_all_models(self, silent=False): if silent or self.confirmDialog(): - index = self.recon_tab_qwidget_settings_layout.count() - 1 + index = self.models_container_widget_layout.count() - 1 while index >= 0: - myWidget = self.recon_tab_qwidget_settings_layout.itemAt( + myWidget = self.models_container_widget_layout.itemAt( index ).widget() if myWidget is not None: @@ -1549,7 +1594,7 @@ def _clear_all_models(self, silent=False): self.pydantic_classes.clear() CONTAINERS_INFO.clear() self.index = 0 - self.build_button.text = "Run Model" + self.reconstruction_run_PushButton.text = "RUN Model" self.prev_model_settings = {} # Displays the json output from the pydantic model UI selections by user @@ -1691,7 +1736,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): # gather input/out locations input_dir = f"{item['input'].value}" - output_dir = f"{item['output'].value}" + output_dir = f"{item['output']}" # build up the arguments for the pydantic model given the current container if cls is None: @@ -1791,9 +1836,8 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): proc_params["config_path"] = str(Path(config_path).absolute()) proc_params["input_path"] = str(Path(input_dir).absolute()) proc_params["output_path"] = str(Path(output_dir).absolute()) - proc_params["output_path_parent"] = str( - Path(output_dir).parent.absolute() - ) + proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) + proc_params["show"] = item["show"] self.addTableEntry( tableID, tableDescToolTip, proc_params @@ -1810,122 +1854,144 @@ def addPollLoop(self, input_data_path, last_time_index): _breakFlag = False while True: time.sleep(10) - + zattrs_data = None try: - data = open_ome_zarr(input_data_path, mode="r") - if "CurrentDimensions" in data.zattrs.keys(): - my_dict1 = data.zattrs["CurrentDimensions"] - sorted_dict_acq = {k: my_dict1[k] for k in sorted(my_dict1, key=lambda x: required_order.index(x))} - my_dict_time_indices_curr = data.zattrs["CurrentDimensions"]["time"] - # print(sorted_dict_acq) - - if "FinalDimensions" in data.zattrs.keys(): - my_dict2 = data.zattrs["FinalDimensions"] - sorted_dict_final = {k: my_dict2[k] for k in sorted(my_dict2, key=lambda x: required_order.index(x))} - # print(sorted_dict_final) - - # use the prev time_index, since this is current acq and we need for other dims to finish acq for this t - # or when all dims match - signifying acq finished - if my_dict_time_indices_curr-2 > last_time_index or json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): - - now = datetime.datetime.now() - ms = now.strftime("%f")[:3] - unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms - - i = 0 - for item in _pydantic_classes: - i += 1 - cls = item["class"] - cls_container = item["container"] - selected_modes = item["selected_modes"] - exclude_modes = item["exclude_modes"] - c_mode_str = item["c_mode_str"] - - # gather input/out locations - input_dir = f"{item['input'].value}" - output_dir = f"{item['output'].value}" - - pydantic_kwargs = {} - pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( - cls_container, cls, pydantic_kwargs, exclude_modes - ) - - input_channel_names, ret_msg = self.clean_string_for_list( - "input_channel_names", pydantic_kwargs["input_channel_names"] - ) - pydantic_kwargs["input_channel_names"] = input_channel_names + try: + data = open_ome_zarr(input_data_path, mode="r") + zattrs_data = data.zattrs + except PermissionError: + pass # On-The-Fly dataset will throw Permission Denied when being written + # Maybe we can read the zaatrs directly in that case + # If this write/read is a constant issue then the zattrs 'CurrentDimensions' key + # should be updated less frequently, instead of current design of updating with + # each image + + if zattrs_data is None: + zattrs_data = self.loadZattrsDirectlyAsDict(input_data_path) + + if zattrs_data is not None: + if "CurrentDimensions" in zattrs_data.keys(): + my_dict1 = zattrs_data["CurrentDimensions"] + sorted_dict_acq = {k: my_dict1[k] for k in sorted(my_dict1, key=lambda x: required_order.index(x))} + my_dict_time_indices_curr = zattrs_data["CurrentDimensions"]["time"] + # print(sorted_dict_acq) + + if "FinalDimensions" in zattrs_data.keys(): + my_dict2 = zattrs_data["FinalDimensions"] + sorted_dict_final = {k: my_dict2[k] for k in sorted(my_dict2, key=lambda x: required_order.index(x))} + # print(sorted_dict_final) + + # use the prev time_index, since this is current acq and we need for other dims to finish acq for this t + # or when all dims match - signifying acq finished + if my_dict_time_indices_curr-2 > last_time_index or json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): + + now = datetime.datetime.now() + ms = now.strftime("%f")[:3] + unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms + + i = 0 + for item in _pydantic_classes: + i += 1 + cls = item["class"] + cls_container = item["container"] + selected_modes = item["selected_modes"] + exclude_modes = item["exclude_modes"] + c_mode_str = item["c_mode_str"] + + # gather input/out locations + input_dir = f"{item['input'].value}" + output_dir = f"{item['output']}" + + pydantic_kwargs = {} + pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( + cls_container, cls, pydantic_kwargs, exclude_modes + ) - if _pollData: - if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): - time_indices = list(range(last_time_index, my_dict_time_indices_curr)) - _breakFlag = True - else: - time_indices = list(range(last_time_index, my_dict_time_indices_curr-2)) - pydantic_kwargs["time_indices"] = time_indices + input_channel_names, ret_msg = self.clean_string_for_list( + "input_channel_names", pydantic_kwargs["input_channel_names"] + ) + pydantic_kwargs["input_channel_names"] = input_channel_names - if "birefringence" in pydantic_kwargs.keys(): - background_path, ret_msg = self.clean_path_string_when_empty( - "background_path", + if _pollData: + if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): + time_indices = list(range(last_time_index, my_dict_time_indices_curr)) + _breakFlag = True + else: + time_indices = list(range(last_time_index, my_dict_time_indices_curr-2)) + pydantic_kwargs["time_indices"] = time_indices + + if "birefringence" in pydantic_kwargs.keys(): + background_path, ret_msg = self.clean_path_string_when_empty( + "background_path", + pydantic_kwargs["birefringence"]["apply_inverse"][ + "background_path" + ], + ) + pydantic_kwargs["birefringence"]["apply_inverse"][ "background_path" - ], - ) - - pydantic_kwargs["birefringence"]["apply_inverse"][ - "background_path" - ] = background_path + ] = background_path - # validate and return errors if None - pydantic_model, ret_msg = self.validate_pydantic_model( - cls, pydantic_kwargs - ) + # validate and return errors if None + pydantic_model, ret_msg = self.validate_pydantic_model( + cls, pydantic_kwargs + ) - # save the yaml files - # path is next to saved data location - save_config_path = str(Path(output_dir).parent.absolute()) - yml_file_name = "-and-".join(selected_modes) - yml_file = yml_file_name + "-" + unique_id + "-{:02d}".format(i) + ".yml" - config_path = os.path.join(save_config_path, yml_file) - utils.model_to_yaml(pydantic_model, config_path) - - expID = "{tID}-{idx}".format(tID=unique_id, idx=i) - tableID = "{tName}: ({tID}-{idx})".format( - tName=c_mode_str, tID=unique_id, idx=i - ) - tableDescToolTip = "{tName}: ({tID}-{idx})".format( - tName=yml_file_name, tID=unique_id, idx=i - ) + # save the yaml files + # path is next to saved data location + save_config_path = str(Path(output_dir).parent.absolute()) + yml_file_name = "-and-".join(selected_modes) + yml_file = yml_file_name + "-" + unique_id + "-{:02d}".format(i) + ".yml" + config_path = os.path.join(save_config_path, yml_file) + utils.model_to_yaml(pydantic_model, config_path) + + expID = "{tID}-{idx}".format(tID=unique_id, idx=i) + tableID = "{tName}: ({tID}-{idx})".format( + tName=c_mode_str, tID=unique_id, idx=i + ) + tableDescToolTip = "{tName}: ({tID}-{idx})".format( + tName=yml_file_name, tID=unique_id, idx=i + ) - proc_params = {} - proc_params["exp_id"] = expID - proc_params["desc"] = tableDescToolTip - proc_params["config_path"] = str(Path(config_path).absolute()) - proc_params["input_path"] = str(Path(input_dir).absolute()) - proc_params["output_path"] = str(Path(output_dir).absolute()) - proc_params["output_path_parent"] = str( - Path(output_dir).parent.absolute() - ) - - tableEntryWorker1 = AddTableEntryWorkerThread(tableID, tableDescToolTip, proc_params) - tableEntryWorker1.add_tableentry_signal.connect(self.addTableEntry) - tableEntryWorker1.start() + proc_params = {} + proc_params["exp_id"] = expID + proc_params["desc"] = tableDescToolTip + proc_params["config_path"] = str(Path(config_path).absolute()) + proc_params["input_path"] = str(Path(input_dir).absolute()) + proc_params["output_path"] = str(Path(output_dir).absolute()) + proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) + proc_params["show"] = False + + tableEntryWorker1 = AddTableEntryWorkerThread(tableID, tableDescToolTip, proc_params) + tableEntryWorker1.add_tableentry_signal.connect(self.addTableEntry) + tableEntryWorker1.start() - if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final) and _breakFlag: + if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final) and _breakFlag: + + tableEntryWorker2 = AddOTFTableEntryWorkerThread(input_data_path, False) + tableEntryWorker2.add_tableOTFentry_signal.connect(self.addRemoveOTFTableEntry) + tableEntryWorker2.start() + + # let child threads finish their work before exiting the parent thread + while tableEntryWorker1.isRunning() or tableEntryWorker2.isRunning(): + time.sleep(1) + time.sleep(5) + break - tableEntryWorker2 = AddOTFTableEntryWorkerThread(input_data_path, False) - tableEntryWorker2.add_tableOTFentry_signal.connect(self.addRemoveOTFTableEntry) - tableEntryWorker2.start() - - # let child threads finish their work before exiting the parent thread - while tableEntryWorker1.isRunning() or tableEntryWorker2.isRunning(): - time.sleep(1) - time.sleep(5) - break - - last_time_index = my_dict_time_indices_curr-2 + last_time_index = my_dict_time_indices_curr-2 except Exception as exc: print(exc.args) + def loadZattrsDirectlyAsDict(self, zattrsFilePathDir): + try: + file_path = os.path.join(zattrsFilePathDir, ".zattrs") + f = open(file_path, "r") + txt = f.read() + f.close() + return json.loads(txt) + except Exception as exc: + print(exc.args) + return None # ======= These function do not implement validation # They simply make the data from GUI translate to input types @@ -2256,19 +2322,15 @@ def get_pydantic_kwargs( # file open/select dialog def _open_file_dialog(self, default_path, type, filter="All Files (*)"): if type == "dir": - return self._open_dialog( - "select a directory", str(default_path), type - ) + return self._open_dialog("select a directory", str(default_path), type, filter) elif type == "file": - return self._open_dialog("select a file", str(default_path), type) + return self._open_dialog("select a file", str(default_path), type, filter) elif type == "files": return self._open_dialog("select file(s)", str(default_path), type, filter) elif type == "save": - return self._open_dialog("save a file", str(default_path), type) + return self._open_dialog("save a file", str(default_path), type, filter) else: - return self._open_dialog( - "select a directory", str(default_path), type - ) + return self._open_dialog("select a directory", str(default_path), type, filter) def _open_dialog(self, title, ref, type, filter="All Files (*)"): """ @@ -2292,7 +2354,7 @@ def _open_dialog(self, title, ref, type, filter="All Files (*)"): ) elif type == "file": path = QFileDialog.getOpenFileName( - None, title, ref, options=options + None, title, ref, filter=filter, options=options )[0] elif type == "files": path = QFileDialog.getOpenFileNames( @@ -2300,7 +2362,7 @@ def _open_dialog(self, title, ref, type, filter="All Files (*)"): )[0] elif type == "save": path = QFileDialog.getSaveFileName( - None, "Choose a save name", ref, options=options + None, "Choose a save name", ref, filter=filter, options=options )[0] else: raise ValueError("Did not understand file dialogue type") @@ -2448,6 +2510,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s proc_params["input_path"] = str(input_data) proc_params["output_path"] = str(output_data) proc_params["output_path_parent"] = str(Path(output_data).parent.absolute()) + proc_params["show"] = False if config_path == "": model = None @@ -2459,7 +2522,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s exclude_modes = item["exclude_modes"] # gather input/out locations - output_dir = f"{item['output'].value}" + output_dir = f"{item['output']}" if output_data == "": output_data = output_dir proc_params["output_path"] = str(output_data) @@ -2575,6 +2638,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s proc_params["input_path"] = "" proc_params["output_path"] = "" proc_params["output_path_parent"] = "" + proc_params["show"] = False tableEntryWorker = AddTableEntryWorkerThread(tableID, tableID, proc_params) tableEntryWorker.add_tableentry_signal.connect(self.tab_recon.addTableEntry) @@ -2771,7 +2835,15 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s def clientRelease(self, expIdx, jobIdx, client_socket, params): # only need to release client from primary job self.JobsMgmt.putJobCompletionInList(True, expIdx, jobIdx) + showData_thread = None if params["primary"]: + if "show" in params: + if params["show"]: + # Read reconstruction data + showData_thread = ShowDataWorkerThread(params["output_path"]) + showData_thread.show_data_signal.connect(self.tab_recon.showDataset) + showData_thread.start() + while not self.JobsMgmt.checkAllExpJobsCompletion(expIdx): time.sleep(1) json_obj = {"uID":expIdx, "jID":jobIdx,"command": "clientRelease"} @@ -2779,11 +2851,14 @@ def clientRelease(self, expIdx, jobIdx, client_socket, params): client_socket.send(json_str.encode()) if self.pool is not None: - print("Number of running threads:", self.pool._work_queue.qsize()) if self.pool._work_queue.qsize() == 0: self.pool.shutdown() self.pool = None + if showData_thread is not None: + while showData_thread.isRunning(): + time.sleep(3) + def runInPool(self, params): if not self.isInitialized: self.initialize() @@ -2888,7 +2963,20 @@ def runInSubProcess(self, params): self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) self.tableUpdateAndCleaupThread() + + +class ShowDataWorkerThread(QThread): + """Worker thread for sending signal for adding component when request comes + from a different thread""" + show_data_signal = pyqtSignal(str) + def __init__(self, path): + super().__init__() + self.path = path + + def run(self): + # Emit the signal to add the widget to the main thread + self.show_data_signal.emit(self.path) class AddOTFTableEntryWorkerThread(QThread): """Worker thread for sending signal for adding component when request comes from a different thread""" diff --git a/setup.cfg b/setup.cfg index f36021c7..9ca61f10 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ install_requires = wget>=3.2 psutil submitit - pydantic>=1.10.17 + pydantic==1.10.19 [options.extras_require] dev = From 7baa16eca03ed33a3db4bc4e85235145b4d7c398 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Thu, 9 Jan 2025 02:52:11 -0500 Subject: [PATCH 22/38] exit polling loop when closing app before finish --- recOrder/plugin/tab_recon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 9b51f201..499d2d87 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1981,6 +1981,8 @@ def addPollLoop(self, input_data_path, last_time_index): last_time_index = my_dict_time_indices_curr-2 except Exception as exc: print(exc.args) + print("Exiting polling for dataset: {data_path}".format(data_path=input_data_path)) + break def loadZattrsDirectlyAsDict(self, zattrsFilePathDir): try: From 949815fbf4bf99cfcea476532019d84c4807dfd9 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Thu, 9 Jan 2025 04:15:23 -0500 Subject: [PATCH 23/38] implemented a Stop method for On-The-Fly polling reconstructions implemented a Stop method for On-The-Fly polling reconstructions if required --- recOrder/plugin/tab_recon.py | 74 ++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 499d2d87..a3aaf501 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1028,21 +1028,53 @@ def addTableEntryJob(self, proc_params): return proc_params - def addRemoveOTFTableEntry(self, OTF_dir_path, bool_msg): - if bool_msg: - otf_label = QLabel(text=OTF_dir_path + " " + _green_dot) - self.proc_OTF_table_QFormLayout.insertRow(0, otf_label) - else: + def addRemoveCheckOTFTableEntry(self, OTF_dir_path, bool_msg, doCheck=False): + if doCheck: try: for row in range(self.proc_OTF_table_QFormLayout.rowCount()): widgetItem = self.proc_OTF_table_QFormLayout.itemAt(row) if widgetItem is not None: - name_widget:QLabel = widgetItem.widget() - name_string = str(name_widget.text()) + name_widget:QWidget = widgetItem.widget() + name_string = str(name_widget.objectName()) if OTF_dir_path in name_string: - self.proc_OTF_table_QFormLayout.removeRow(row) + for item in name_widget.findChildren(QPushButton): + _poll_Stop_PushButton:QPushButton = item + return _poll_Stop_PushButton.isChecked() + return False except Exception as exc: print(exc.args) + return False + else: + if bool_msg: + _poll_otf_label = QLabel(text=OTF_dir_path + " " + _green_dot) + _poll_Stop_PushButton = QPushButton("Stop") + _poll_Stop_PushButton.setCheckable(True) # Make the button checkable + _poll_Stop_PushButton.clicked.connect(lambda:self.stopOTF_PushButtonCall(_poll_otf_label, OTF_dir_path + " " + _red_dot)) + + _poll_data_widget = QWidget() + _poll_data_widget.setObjectName(OTF_dir_path) + _poll_data_widget_layout = QHBoxLayout() + _poll_data_widget.setLayout(_poll_data_widget_layout) + _poll_data_widget_layout.addWidget(_poll_otf_label) + _poll_data_widget_layout.addWidget(_poll_Stop_PushButton) + + self.proc_OTF_table_QFormLayout.insertRow(0, _poll_data_widget) + else: + try: + for row in range(self.proc_OTF_table_QFormLayout.rowCount()): + widgetItem = self.proc_OTF_table_QFormLayout.itemAt(row) + if widgetItem is not None: + name_widget:QWidget = widgetItem.widget() + name_string = str(name_widget.objectName()) + if OTF_dir_path in name_string: + self.proc_OTF_table_QFormLayout.removeRow(row) + except Exception as exc: + print(exc.args) + + def stopOTF_PushButtonCall(self, label, txt): + _poll_otf_label: QLabel = label + _poll_otf_label.setText(txt) + self.setDisabled(True) # adds processing entry to _qwidgetTabEntry_layout as row item # row item will be purged from table as processing finishes @@ -1848,14 +1880,25 @@ def addPollLoop(self, input_data_path, last_time_index): required_order = ['time', 'position', 'z', 'channel'] _pollData = True - tableEntryWorker = AddOTFTableEntryWorkerThread(input_data_path, True) - tableEntryWorker.add_tableOTFentry_signal.connect(self.addRemoveOTFTableEntry) + tableEntryWorker = AddOTFTableEntryWorkerThread(input_data_path, True, False) + tableEntryWorker.add_tableOTFentry_signal.connect(self.addRemoveCheckOTFTableEntry) tableEntryWorker.start() _breakFlag = False while True: time.sleep(10) zattrs_data = None try: + _stopCalled = self.addRemoveCheckOTFTableEntry(input_data_path, True, doCheck=True) + if _stopCalled: + tableEntryWorker2 = AddOTFTableEntryWorkerThread(input_data_path, False, False) + tableEntryWorker2.add_tableOTFentry_signal.connect(self.addRemoveCheckOTFTableEntry) + tableEntryWorker2.start() + + # let child threads finish their work before exiting the parent thread + while tableEntryWorker2.isRunning(): + time.sleep(1) + time.sleep(5) + break try: data = open_ome_zarr(input_data_path, mode="r") zattrs_data = data.zattrs @@ -1968,8 +2011,8 @@ def addPollLoop(self, input_data_path, last_time_index): if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final) and _breakFlag: - tableEntryWorker2 = AddOTFTableEntryWorkerThread(input_data_path, False) - tableEntryWorker2.add_tableOTFentry_signal.connect(self.addRemoveOTFTableEntry) + tableEntryWorker2 = AddOTFTableEntryWorkerThread(input_data_path, False, False) + tableEntryWorker2.add_tableOTFentry_signal.connect(self.addRemoveCheckOTFTableEntry) tableEntryWorker2.start() # let child threads finish their work before exiting the parent thread @@ -2982,16 +3025,17 @@ def run(self): class AddOTFTableEntryWorkerThread(QThread): """Worker thread for sending signal for adding component when request comes from a different thread""" - add_tableOTFentry_signal = pyqtSignal(str, bool) + add_tableOTFentry_signal = pyqtSignal(str, bool, bool) - def __init__(self, OTF_dir_path, bool_msg): + def __init__(self, OTF_dir_path, bool_msg, doCheck=False): super().__init__() self.OTF_dir_path = OTF_dir_path self.bool_msg = bool_msg + self.doCheck = doCheck def run(self): # Emit the signal to add the widget to the main thread - self.add_tableOTFentry_signal.emit(self.OTF_dir_path, self.bool_msg) + self.add_tableOTFentry_signal.emit(self.OTF_dir_path, self.bool_msg, self.doCheck) class AddTableEntryWorkerThread(QThread): """Worker thread for sending signal for adding component when request comes from a different thread""" From 2c58a211ebdb0114e3a1dd09207d146772f5d729 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Fri, 10 Jan 2025 14:27:27 -0500 Subject: [PATCH 24/38] GUI related - added info icon next to Input Store label when a dataset path is set which displays channel names for convenience, more metadata could be displayed that might be relevant for reconstruction - removed line widget at end of each model container to make scrollbar more apparent --- recOrder/plugin/tab_recon.py | 54 +++++++++++++++++------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index a3aaf501..06b1a424 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -86,6 +86,7 @@ _validate_ok = "✔️" _green_dot = "🟢" _red_dot = "🔴" +_info_icon = "ⓘ" # For now replicate CLI processing modes - these could reside in the CLI settings file as well # for consistency @@ -134,6 +135,7 @@ def __init__(self, parent=None, stand_alone=False): self.input_directory_dataset = None self.input_directory_datasetMeta = None + self.input_channel_names = [] # Parent (Widget) which holds the GUI ############################## self.recon_tab_mainScrollArea = QScrollArea() @@ -466,10 +468,15 @@ def validateInputData( # Sort and validate the input paths, expanding plates into lists of positions # return True, MSG_SUCCESS try: + self.input_channel_names = [] + self.data_input_Label.value = "Input Store" input_paths = Path(input_data_folder) with open_ome_zarr(input_paths, mode="r") as dataset: # ToDo: Metadata reading and implementation in GUI for # channel names, time indicies, etc. + self.input_channel_names = dataset.channel_names + self.data_input_Label.value = "Input Store" + " " + _info_icon + self.data_input_Label.tooltip = "Channel Names:\n- " + "\n- ".join(self.input_channel_names) if not BG and metadata: self.input_directory_dataset = dataset @@ -1316,17 +1323,6 @@ def _create_acq_contols2( self.messageBox(ret_msg) return - # Line seperator between pydantic UI components - _line = QFrame() - _line.setMinimumWidth(1) - _line.setFixedHeight(2) - _line.setFrameShape(QFrame.HLine) - _line.setFrameShadow(QFrame.Sunken) - _line.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Minimum) - _line.setStyleSheet( - "border:1px solid rgb(128,128,128); border-width: 1px;" - ) - # PushButton to delete a UI container # Use case when a wrong selection of input modes get selected eg Bire+Fl # Preferably this root level validation should occur before values arevalidated @@ -1363,6 +1359,9 @@ def _create_acq_contols2( lambda: self.readAndSetOutputPathOnValidation(_output_data_loc, _output_data_btn, save_path) ) + _show_CheckBox = widgets.CheckBox(name="Show after Reconstruction", value=True) + _validate_button = widgets.PushButton(name="Validate") + # Passing all UI components that would be deleted _expandingTabEntryWidget = QWidget() _del_button.clicked.connect( @@ -1371,9 +1370,9 @@ def _create_acq_contols2( recon_pydantic_container.native, _output_data_loc.native, _output_data_btn.native, + _show_CheckBox.native, + _validate_button.native, _del_button.native, - _line, - _idx, _str, ) ) @@ -1405,9 +1404,7 @@ def _create_acq_contols2( _collapsibleBoxWidget = CollapsibleBox( c_mode_str ) # tableEntryID, tableEntryShortDesc - should update with processing status - - _show_CheckBox = widgets.CheckBox(name="Show after Reconstruction", value=True) - _validate_button = widgets.PushButton(name="Validate") + _validate_button.clicked.connect(lambda:self._validate_model(_str, _collapsibleBoxWidget)) _hBox_widget2 = QWidget() @@ -1433,11 +1430,10 @@ def _create_acq_contols2( ) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget2) - _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_line) _scrollAreaCollapsibleBox.setMinimumHeight(_scrollAreaCollapsibleBoxWidgetLayout.sizeHint().height()) _collapsibleBoxWidget.setSizePolicy( - QSizePolicy.Expanding, QSizePolicy.Fixed + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed ) _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) @@ -1578,21 +1574,21 @@ def _validate_model(self, _str, _collapsibleBoxWidget): # UI components deletion - maybe just needs the parent container instead of individual components - def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, index, _str): + def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, wid6, _str): if not self.confirmDialog(): return False - if wid5 is not None: - wid5.setParent(None) - if wid4 is not None: - wid4.setParent(None) - if wid3 is not None: - wid3.setParent(None) - if wid2 is not None: - wid2.setParent(None) - if wid1 is not None: - wid1.setParent(None) + # if wid5 is not None: + # wid5.setParent(None) + # if wid4 is not None: + # wid4.setParent(None) + # if wid3 is not None: + # wid3.setParent(None) + # if wid2 is not None: + # wid2.setParent(None) + # if wid1 is not None: + # wid1.setParent(None) if wid0 is not None: wid0.setParent(None) From 735aec74b63214881814139d1dba6ce32e772b39 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Fri, 10 Jan 2025 15:44:33 -0500 Subject: [PATCH 25/38] create logs dir if it does not exist - we are only reading the location here but it needs to exist, the CLI is creating the logs based on the path --- recOrder/cli/jobs_mgmt.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py index fc0ed07a..1f35f289 100644 --- a/recOrder/cli/jobs_mgmt.py +++ b/recOrder/cli/jobs_mgmt.py @@ -21,7 +21,15 @@ def clearLogs(self): thread = threading.Thread(target=self.clearLogFiles, args={self.logsPath,}) thread.start() + def create_dir_if_not_exists(self, directory): + if not os.path.exists(directory): + os.makedirs(directory) + time.sleep(1) + def clearLogFiles(self, dirPath, silent=True): + + self.create_dir_if_not_exists(dirPath) + for filename in os.listdir(dirPath): file_path = os.path.join(dirPath, filename) try: From 1c51eead61e387932166ec127cdf3acd9c24932b Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sat, 11 Jan 2025 17:11:48 -0500 Subject: [PATCH 26/38] fixes output path not setting correctly --- recOrder/plugin/tab_recon.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 06b1a424..b9163619 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1453,6 +1453,7 @@ def _create_acq_contols2( "class": pydantic_class, "input": self.data_input_LineEdit, "output": os.path.join(Path(self.output_directory).absolute(), _output_data_loc.value), + "output_parent_dir": str(Path(self.output_directory).absolute()), "output_LineEdit": _output_data_loc, "output_Button": _output_data_btn, "container": recon_pydantic_container, @@ -1761,10 +1762,14 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): selected_modes = item["selected_modes"] exclude_modes = item["exclude_modes"] c_mode_str = item["c_mode_str"] + output_LineEdit = item["output_LineEdit"] + output_parent_dir = item["output_parent_dir"] + + full_out_path = os.path.join(output_parent_dir, output_LineEdit.value) # gather input/out locations input_dir = f"{item['input'].value}" - output_dir = f"{item['output']}" + output_dir = full_out_path # build up the arguments for the pydantic model given the current container if cls is None: @@ -1936,10 +1941,13 @@ def addPollLoop(self, input_data_path, last_time_index): selected_modes = item["selected_modes"] exclude_modes = item["exclude_modes"] c_mode_str = item["c_mode_str"] + output_LineEdit = item["output_LineEdit"] + output_parent_dir = item["output_parent_dir"] + full_out_path = os.path.join(output_parent_dir, output_LineEdit.value) # gather input/out locations input_dir = f"{item['input'].value}" - output_dir = f"{item['output']}" + output_dir = full_out_path pydantic_kwargs = {} pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( @@ -2561,9 +2569,12 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s cls = item["class"] cls_container = item["container"] exclude_modes = item["exclude_modes"] + output_LineEdit = item["output_LineEdit"] + output_parent_dir = item["output_parent_dir"] + full_out_path = os.path.join(output_parent_dir, output_LineEdit.value) # gather input/out locations - output_dir = f"{item['output']}" + output_dir = full_out_path if output_data == "": output_data = output_dir proc_params["output_path"] = str(output_data) From ca1ae0e2a45dddf213e87ac89a0a55c68cd20084 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sat, 11 Jan 2025 18:40:46 -0500 Subject: [PATCH 27/38] added pixel size meta to info icon --- recOrder/plugin/tab_recon.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index b9163619..596b5a6c 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -474,9 +474,27 @@ def validateInputData( with open_ome_zarr(input_paths, mode="r") as dataset: # ToDo: Metadata reading and implementation in GUI for # channel names, time indicies, etc. - self.input_channel_names = dataset.channel_names - self.data_input_Label.value = "Input Store" + " " + _info_icon - self.data_input_Label.tooltip = "Channel Names:\n- " + "\n- ".join(self.input_channel_names) + try: + self.input_channel_names = dataset.channel_names + self.data_input_Label.value = "Input Store" + " " + _info_icon + self.data_input_Label.tooltip = "Channel Names:\n- " + "\n- ".join(self.input_channel_names) + except Exception as exc: + print(exc.args) + + try: + for path, pos in dataset.positions(): + axes = pos.zgroup.attrs["multiscales"][0]["axes"] + string_array_n = [str(x["name"]) for x in axes] + string_array = [str(x) for x in pos.zgroup.attrs["multiscales"][0]["datasets"][0]["coordinateTransformations"][0]["scale"]] + string_scale = [] + for i in range(len(string_array_n)): + string_scale.append("{n}={d}".format(n=string_array_n[i], d=string_array[i])) + txt = "\n\nScale: " + ", ".join(string_scale) + self.data_input_Label.tooltip += txt + break + except Exception as exc: + print(exc.args) + if not BG and metadata: self.input_directory_dataset = dataset From 420da1dae32db494a372f207a155fb8eb192974a Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sun, 12 Jan 2025 02:07:49 -0500 Subject: [PATCH 28/38] update output dir in defined models when changed --- recOrder/plugin/tab_recon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 596b5a6c..6b4e2f9a 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -566,6 +566,7 @@ def validateModelOutputPaths(self): for model_item in self.pydantic_classes: output_LineEdit = model_item["output_LineEdit"] output_Button = model_item["output_Button"] + model_item["output_parent_dir"] = self.output_directory full_out_path = os.path.join(Path(self.output_directory).absolute(), output_LineEdit.value) model_item["output"] = full_out_path From 159f0bea14b25c1d33d29f91e714143f7bfb9b4a Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sun, 12 Jan 2025 03:17:07 -0500 Subject: [PATCH 29/38] make on-the-fly entry scrollable and not block resizing --- recOrder/plugin/tab_recon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 6b4e2f9a..060e5200 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1072,7 +1072,7 @@ def addRemoveCheckOTFTableEntry(self, OTF_dir_path, bool_msg, doCheck=False): return False else: if bool_msg: - _poll_otf_label = QLabel(text=OTF_dir_path + " " + _green_dot) + _poll_otf_label = ScrollableLabel(text=OTF_dir_path + " " + _green_dot) _poll_Stop_PushButton = QPushButton("Stop") _poll_Stop_PushButton.setCheckable(True) # Make the button checkable _poll_Stop_PushButton.clicked.connect(lambda:self.stopOTF_PushButtonCall(_poll_otf_label, OTF_dir_path + " " + _red_dot)) From d9a39c384634a99a5f816e517e6e3dab212e6810 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sun, 12 Jan 2025 03:47:55 -0500 Subject: [PATCH 30/38] added script to simulate a "fake" recOrder acquisition - script to test on-the-fly reconstruction POC --- recOrder/scripts/simulate_zarr_acq.py | 136 ++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 recOrder/scripts/simulate_zarr_acq.py diff --git a/recOrder/scripts/simulate_zarr_acq.py b/recOrder/scripts/simulate_zarr_acq.py new file mode 100644 index 00000000..b13088da --- /dev/null +++ b/recOrder/scripts/simulate_zarr_acq.py @@ -0,0 +1,136 @@ +from pathlib import Path +from iohub.convert import TIFFConverter +from iohub.ngff import open_ome_zarr +from recOrder.cli.utils import create_empty_hcs_zarr + +import time, threading, os, shutil, json + +# This script is a demo .zarr acquisition simulation from an acquired .zarr store +# The script copies and writes additional metadata to .zattrs inserting two keys +# The two keys are "FinalDimensions" and "CurrentDimensions". +# The "FinalDimensions" key with (t,p,z,c) needs to be inserted when the dataset is created +# and then should be updated at close to ensure aborted acquisitions represent correct dimensions. +# The "CurrentDimensions" key should have the same (t,p,z,c) information and should be written out +# either with every new image, end of dimension OR at frequent intervals. +# Refer further notes below in the example regarding encountered issues. +# +# Refer to steps at the end of the file on steps to run this file + +#%% ############################################# +def convertData(tif_path, latest_out_path, prefix="", data_type_str="ometiff"): + converter = TIFFConverter( + os.path.join(tif_path , prefix), + latest_out_path, + data_type=data_type_str, + grid_layout=False, + ) + converter.run() + +def runConvert(ome_tif_path): + out_path = os.path.join(Path(ome_tif_path).parent.absolute(), ("raw_" + Path(ome_tif_path).name + ".zarr")) + convertData(ome_tif_path, out_path) + +#%% ############################################# + +def runAcq(input_path="", waitBetweenT=30): + + output_store_path = os.path.join(Path(input_path).parent.absolute(), ("acq_sim_" + Path(input_path).name)) + + if Path(output_store_path).exists(): + shutil.rmtree(output_store_path) + time.sleep(1) + + input_data = open_ome_zarr(input_path, mode="r") + channel_names = input_data.channel_names + + position_keys: list[tuple[str]] = [] + + for path, pos in input_data.positions(): + shape = pos["0"].shape + dtype = pos["0"].dtype + chunks = pos["0"].chunks + scale = (1, 1, 1, 1, 1) + position_keys.append(path.split("/")) + + create_empty_hcs_zarr( + output_store_path, + position_keys, + shape, + chunks, + scale, + channel_names, + dtype, + {}, + ) + output_dataset = open_ome_zarr(output_store_path, mode="r+") + + if "Summary" in input_data.zattrs.keys(): + output_dataset.zattrs["Summary"] = input_data.zattrs["Summary"] + + output_dataset.zattrs.update({"FinalDimensions": { + "channel": shape[1], + "position": len(position_keys), + "time": shape[0], + "z": shape[2] + } + }) + + total_time = shape[0] + total_pos = len(position_keys) + total_z = shape[2] + total_c = shape[1] + for t in range(total_time): + for p in range(total_pos): + for z in range(total_z): + for c in range(total_c): + position_key_string = "/".join(position_keys[p]) + img_src = input_data[position_key_string][0][t, c, z] + + img_data = output_dataset[position_key_string][0] + img_data[t, c, z] = img_src + + # Note: On-The-Fly dataset reconstruction will throw Permission Denied when being written + # Maybe we can read the zaatrs directly in that case as a file which is less blocking + # If this write/read is a constant issue then the zattrs 'CurrentDimensions' key + # should be updated less frequently, instead of current design of updating with + # each image + output_dataset.zattrs.update({"CurrentDimensions": { + "channel": total_c, + "position": p+1, + "time": t+1, + "z": z+1 + } + }) + + required_order = ['time', 'position', 'z', 'channel'] + my_dict = output_dataset.zattrs["CurrentDimensions"] + sorted_dict_acq = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} + print("Writer thread - Acquisition Dim:", sorted_dict_acq) + time.sleep(waitBetweenT) # sleep after every t + + output_dataset.close + +#%% ############################################# +def runAcquire(input_path, waitBetweenT): + runThread1Acq = threading.Thread(target=runAcq, args=(input_path, waitBetweenT)) + runThread1Acq.start() + +#%% ############################################# +# Step 1: +# Convert an existing ome-tif recOrder acquisition, preferably with all dims (t, p, z, c) +# This will convert an existing ome-tif to a .zarr storage + +# ome_tif_path = "/ome-zarr_data/recOrderAcq/test/snap_6D_ometiff_1" +# runConvert(ome_tif_path) + +#%% ############################################# +# Step 2: +# run the test to simulate Acquiring a recOrder .zarr store + +input_path = "/ome-zarr_data/recOrderAcq/test/raw_snap_6D_ometiff_1.zarr" +waitBetweenT = 90 +runAcquire(input_path, waitBetweenT) + + + + From 4ef11a1de8bc6688aacf8e4493ec7761e0ddada0 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sun, 12 Jan 2025 04:17:14 -0500 Subject: [PATCH 31/38] fix for checking output path existing --- recOrder/plugin/tab_recon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 060e5200..595cecf6 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1361,7 +1361,7 @@ def _create_acq_contols2( # These could be multiple based on user selection for each model # Inherits from Input by default at creation time name_without_ext = os.path.splitext(self.input_directory)[0] - save_path = os.path.join(Path(self.output_directory).parent.absolute(), (name_without_ext + ("_"+c_mode_short+"_"+num_str) + ".zarr")) + save_path = os.path.join(Path(self.output_directory).absolute(), (name_without_ext + ("_"+c_mode_short+"_"+num_str) + ".zarr")) save_path_exists = True if Path(save_path).exists() else False _output_data_loc = widgets.LineEdit( value=Path(save_path).name, tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") From 8ee6c056db5de716e2c7683aadd9aef9a3e397a1 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sun, 12 Jan 2025 04:54:52 -0500 Subject: [PATCH 32/38] fix for checking output path existing --- recOrder/plugin/tab_recon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 595cecf6..ef9c3949 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1360,7 +1360,7 @@ def _create_acq_contols2( # Output Data location # These could be multiple based on user selection for each model # Inherits from Input by default at creation time - name_without_ext = os.path.splitext(self.input_directory)[0] + name_without_ext = os.path.splitext(Path(self.input_directory).name)[0] save_path = os.path.join(Path(self.output_directory).absolute(), (name_without_ext + ("_"+c_mode_short+"_"+num_str) + ".zarr")) save_path_exists = True if Path(save_path).exists() else False _output_data_loc = widgets.LineEdit( From 23d4cfc1a703fa7694855652ba79195f56a8b406 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 14 Jan 2025 03:40:31 -0500 Subject: [PATCH 33/38] logs folder to be created besides dataset - logs folder will reside next to the output dataset - fixed an issue with reconstructed data showing irrespective of selection --- .../cli/apply_inverse_transfer_function.py | 8 ++- recOrder/cli/jobs_mgmt.py | 56 ++++++------------- recOrder/plugin/tab_recon.py | 22 ++++---- 3 files changed, 34 insertions(+), 52 deletions(-) diff --git a/recOrder/cli/apply_inverse_transfer_function.py b/recOrder/cli/apply_inverse_transfer_function.py index 7e658ad4..4ace46ad 100644 --- a/recOrder/cli/apply_inverse_transfer_function.py +++ b/recOrder/cli/apply_inverse_transfer_function.py @@ -3,6 +3,7 @@ from functools import partial from pathlib import Path +import os import click import numpy as np import torch @@ -342,7 +343,10 @@ def apply_inverse_transfer_function_cli( f"{cpu_request} CPU{'s' if cpu_request > 1 else ''} and " f"{gb_ram_request} GB of memory per CPU." ) - executor = submitit.AutoExecutor(folder="logs") + + name_without_ext = os.path.splitext(Path(output_dirpath).name)[0] + executor_folder = os.path.join(Path(output_dirpath).parent.absolute(), name_without_ext + "_logs") + executor = submitit.AutoExecutor(folder=Path(executor_folder)) executor.update_parameters( slurm_array_parallelism=np.min([50, num_jobs]), @@ -378,7 +382,7 @@ def apply_inverse_transfer_function_cli( job : submitit.Job = j job_idx : str = job.job_id position = input_position_dirpaths[i] - JM.putJobInList(job, unique_id, str(job_idx), position) + JM.putJobInList(job, unique_id, str(job_idx), position, str(executor.folder.absolute())) i += 1 JM.setShorterTimeout() diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py index 1f35f289..3f805c29 100644 --- a/recOrder/cli/jobs_mgmt.py +++ b/recOrder/cli/jobs_mgmt.py @@ -1,4 +1,5 @@ -import os, json, shutil +import os, json +from pathlib import Path import socket import submitit import threading, time @@ -13,47 +14,24 @@ class JobsManagement(): def __init__(self, *args, **kwargs): self.executor = submitit.AutoExecutor(folder="logs") - self.logsPath = self.executor.folder self.clientsocket = None self.uIDsjobIDs = {} # uIDsjobIDs[uid][jid] = job - def clearLogs(self): - thread = threading.Thread(target=self.clearLogFiles, args={self.logsPath,}) - thread.start() + def checkForJobIDFile(self, jobID, logsPath, extension="out"): - def create_dir_if_not_exists(self, directory): - if not os.path.exists(directory): - os.makedirs(directory) - time.sleep(1) - - def clearLogFiles(self, dirPath, silent=True): - - self.create_dir_if_not_exists(dirPath) - - for filename in os.listdir(dirPath): - file_path = os.path.join(dirPath, filename) + if Path(logsPath).exists(): + files = os.listdir(logsPath) try: - if os.path.isfile(file_path) or os.path.islink(file_path): - os.unlink(file_path) - elif os.path.isdir(file_path): - shutil.rmtree(file_path) - except Exception as e: - if not silent: - print('Failed to delete %s. Reason: %s' % (file_path, e)) - - def checkForJobIDFile(self, jobID, extension="out"): - files = os.listdir(self.logsPath) - try: - for file in files: - if file.endswith(extension): - if jobID in file: - file_path = os.path.join(self.logsPath, file) - f = open(file_path, "r") - txt = f.read() - f.close() - return txt - except Exception as exc: - print(exc.args) + for file in files: + if file.endswith(extension): + if jobID in file: + file_path = os.path.join(logsPath, file) + f = open(file_path, "r") + txt = f.read() + f.close() + return txt + except Exception as exc: + print(exc.args) return "" def setShorterTimeout(self): @@ -137,7 +115,7 @@ def putJobCompletionInList(self, jobBool, uID: str, jID: str, mode="client"): if jID in self.uIDsjobIDs[uID].keys(): self.uIDsjobIDs[uID][jID] = jobBool - def putJobInList(self, job, uID: str, jID: str, well:str, mode="client"): + def putJobInList(self, job, uID: str, jID: str, well:str, log_folder_path:str="", mode="client"): try: well = str(well) if ".zarr" in well: @@ -150,7 +128,7 @@ def putJobInList(self, job, uID: str, jID: str, well:str, mode="client"): else: if jID not in self.uIDsjobIDs[uID].keys(): self.uIDsjobIDs[uID][jID] = job - json_obj = {uID:{"jID": str(jID), "pos": well}} + json_obj = {uID:{"jID": str(jID), "pos": well, "log": log_folder_path}} json_str = json.dumps(json_obj)+"\n" self.clientsocket.send(json_str.encode()) else: diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index ef9c3949..8326d74a 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1,4 +1,4 @@ -import sys, os, json, subprocess, time, datetime, uuid +import os, json, subprocess, time, datetime, uuid import socket, threading from pathlib import Path @@ -1479,7 +1479,7 @@ def _create_acq_contols2( "selected_modes": selected_modes.copy(), "exclude_modes": exclude_modes.copy(), "poll_data": self.pollData, - "show": _show_CheckBox.value, + "show": _show_CheckBox, } ) self.index += 1 @@ -1889,7 +1889,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): proc_params["input_path"] = str(Path(input_dir).absolute()) proc_params["output_path"] = str(Path(output_dir).absolute()) proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) - proc_params["show"] = item["show"] + proc_params["show"] = item["show"].value self.addTableEntry( tableID, tableDescToolTip, proc_params @@ -2466,7 +2466,6 @@ def initialize(self): self.workerThreadRowDeletion = RowDeletionWorkerThread(self.formLayout) self.workerThreadRowDeletion.removeRowSignal.connect(self.tab_recon.removeRow) self.workerThreadRowDeletion.start() - self.JobsMgmt.clearLogs() self.isInitialized = True def setNewInstances(self, formLayout, tab_recon, parentForm): @@ -2505,7 +2504,7 @@ def startServer(self): break try: # dont block the server thread - thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=("", "", "", client_socket),) + thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=("", "", "", "", client_socket),) thread.start() except Exception as exc: print(exc.args) @@ -2544,7 +2543,7 @@ def shutDownPool(self): # on errors - table row item is updated but there is no row deletion # on successful processing - the row item is expected to be deleted # row is being deleted from a seperate thread for which we need to connect using signal - def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_socket=None): + def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_folder_path="", client_socket=None): # finished will be updated by the job - submitit status jobIdx = str(jobIdx) if client_socket is not None and expIdx=="" and jobIdx=="": @@ -2676,13 +2675,14 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s expIdx = k jobIdx = json_obj[k]["jID"] wellName = json_obj[k]["pos"] + logs_folder_path = json_obj[k]["log"] if expIdx not in self.results.keys(): # this job came from agnostic CLI route - no processing now = datetime.datetime.now() ms = now.strftime("%f")[:3] unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms expIdx = expIdx +"-"+ unique_id self.JobsMgmt.putJobInList(None, expIdx, str(jobIdx), wellName, mode="server") - thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=(expIdx, jobIdx, wellName, client_socket)) + thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=(expIdx, jobIdx, wellName, logs_folder_path, client_socket)) thread.start() return except: @@ -2805,7 +2805,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s break elif params["status"] in [STATUS_errored_job]: jobERR = self.JobsMgmt.checkForJobIDFile( - jobIdx, extension="err" + jobIdx, logs_folder_path, extension="err" ) _infoBox.setText( jobIdx + "\n" + params["desc"] + "\n\n" + jobERR @@ -2814,7 +2814,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s break else: jobTXT = self.JobsMgmt.checkForJobIDFile( - jobIdx, extension="out" + jobIdx, logs_folder_path, extension="out" ) try: if jobTXT == "": # job file not created yet @@ -2843,7 +2843,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s elif JOB_TRIGGERED_EXC in jobTXT: params["status"] = STATUS_errored_job jobERR = self.JobsMgmt.checkForJobIDFile( - jobIdx, extension="err" + jobIdx, logs_folder_path, extension="err" ) _infoBox.setText( jobIdx @@ -2875,7 +2875,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", client_s break else: jobERR = self.JobsMgmt.checkForJobIDFile( - jobIdx, extension="err" + jobIdx, logs_folder_path, extension="err" ) _infoBox.setText( jobIdx From b6c59366f9ef860332bce52b1c307fcb78ed79c1 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Thu, 16 Jan 2025 01:39:29 -0500 Subject: [PATCH 34/38] display SLURM related errors if Jobs output txt is empty --- recOrder/plugin/tab_recon.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 8326d74a..11ac964a 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -2819,6 +2819,21 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol try: if jobTXT == "": # job file not created yet time.sleep(2) + _tUpdateCount += 2 + if _tUpdateCount > 10: # if out file is empty for 10s, check the err file to update user + jobERR = self.JobsMgmt.checkForJobIDFile( + jobIdx, logs_folder_path, extension="err" + ) + _infoBox.setText( + jobIdx + + "\n" + + params["desc"] + + "\n\n" + + jobERR + ) + if _tUpdateCount > _tUpdateCountTimeout: + self.clientRelease(expIdx, jobIdx, client_socket, params) + break elif ( params["status"] == STATUS_finished_job From 796cb59fb42ef2a91d0f2388f9c5b90d6e167219 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Thu, 16 Jan 2025 05:03:37 -0500 Subject: [PATCH 35/38] top scrollbar for model, container sizing now does not need second vert scrollbar --- recOrder/plugin/tab_recon.py | 45 ++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 11ac964a..4f1ee11d 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1144,9 +1144,7 @@ def addTableEntry( _expandingTabEntryWidget = QWidget() _expandingTabEntryWidget.toolTip = tableEntryShortDesc _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) + _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) proc_params["tableEntryID"] = tableEntryID proc_params["parent_layout"] = _scrollAreaCollapsibleBoxWidgetLayout @@ -1408,7 +1406,7 @@ def _create_acq_contols2( _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - _scrollAreaCollapsibleBoxWidget = QWidget() + _scrollAreaCollapsibleBoxWidget = MyWidget() _scrollAreaCollapsibleBoxWidget.setLayout( _scrollAreaCollapsibleBoxWidgetLayout ) @@ -1418,6 +1416,13 @@ def _create_acq_contols2( _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) _collapsibleBoxWidgetLayout = QVBoxLayout() + _collapsibleBoxWidgetLayout.setAlignment(Qt.AlignmentFlag.AlignTop) + + scrollbar = _scrollAreaCollapsibleBox.horizontalScrollBar() + _scrollAreaCollapsibleBoxWidget.resized.connect(lambda:self.check_scrollbar_visibility(scrollbar)) + + _scrollAreaCollapsibleBoxWidgetLayout.addWidget(scrollbar, alignment=Qt.AlignmentFlag.AlignTop) # Place at the top + _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) _collapsibleBoxWidget = CollapsibleBox( @@ -1435,30 +1440,25 @@ def _create_acq_contols2( _hBox_layout2.addWidget(_del_button.native) _expandingTabEntryWidgetLayout = QVBoxLayout() + _expandingTabEntryWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _expandingTabEntryWidgetLayout.addWidget(_collapsibleBoxWidget) _expandingTabEntryWidget.toolTip = c_mode_str _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.setSizePolicy( - QSizePolicy.Expanding, QSizePolicy.Fixed - ) + _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) _expandingTabEntryWidget.layout().setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - _scrollAreaCollapsibleBoxWidgetLayout.addWidget( - recon_pydantic_container.native - ) + _scrollAreaCollapsibleBoxWidgetLayout.addWidget(recon_pydantic_container.native) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget2) - _scrollAreaCollapsibleBox.setMinimumHeight(_scrollAreaCollapsibleBoxWidgetLayout.sizeHint().height()) - _collapsibleBoxWidget.setSizePolicy( - QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed - ) + _scrollAreaCollapsibleBox.setMinimumHeight(_scrollAreaCollapsibleBoxWidgetLayout.sizeHint().height()+20) + _collapsibleBoxWidget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) self.models_container_widget_layout.addWidget( _expandingTabEntryWidget - ) + ) # Store a copy of the pydantic container along with all its associated components and properties # We dont needs a copy of the class but storing for now @@ -1492,6 +1492,12 @@ def _create_acq_contols2( self.reconstruction_run_PushButton.text = "RUN Model" return pydantic_model + + def check_scrollbar_visibility(self, scrollbar): + h_scrollbar = scrollbar + + # Hide scrollbar if not needed + h_scrollbar.setVisible(h_scrollbar.maximum() > h_scrollbar.minimum()) def _validate_model(self, _str, _collapsibleBoxWidget): i = 0 @@ -3207,6 +3213,15 @@ def __init__(self, text, *args, **kwargs): def setText(self, text): self.label.setText(text) +class MyWidget(QWidget): + resized = pyqtSignal() + + def __init__(self): + super().__init__() + + def resizeEvent(self, event): + self.resized.emit() + super().resizeEvent(event) class CollapsibleBox(QWidget): """A collapsible widget""" From 282c4d5342143982b136c495816ffdb0e744c1f5 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Sat, 18 Jan 2025 06:30:25 -0500 Subject: [PATCH 36/38] multi-pos bugfix + multiple enhancements fixes: - multi-pos dataset would be displayed after single pos processing enhancements: - CLI will print Job status when used as cmd line, not for GUI - use single socket connection when multi-pos is spawned by a request - added "rx" field to model-container - minor GUI tweaks --- .../cli/apply_inverse_transfer_function.py | 5 +- recOrder/cli/jobs_mgmt.py | 60 +++++++++------ recOrder/cli/monitor.py | 64 ++++++++-------- recOrder/plugin/tab_recon.py | 74 +++++++++++++------ 4 files changed, 126 insertions(+), 77 deletions(-) diff --git a/recOrder/cli/apply_inverse_transfer_function.py b/recOrder/cli/apply_inverse_transfer_function.py index 4ace46ad..c4172735 100644 --- a/recOrder/cli/apply_inverse_transfer_function.py +++ b/recOrder/cli/apply_inverse_transfer_function.py @@ -375,6 +375,7 @@ def apply_inverse_transfer_function_cli( f"{num_jobs} job{'s' if num_jobs > 1 else ''} submitted {'locally' if executor.cluster == 'local' else 'via ' + executor.cluster}." ) + doPrint = True # CLI prints Job status when used as cmd line if unique_id != "": # no unique_id means no job submission info being listened to JM.startClient() i=0 @@ -384,9 +385,11 @@ def apply_inverse_transfer_function_cli( position = input_position_dirpaths[i] JM.putJobInList(job, unique_id, str(job_idx), position, str(executor.folder.absolute())) i += 1 + JM.sendDataThread() JM.setShorterTimeout() + doPrint = False # CLI printing disabled when using GUI - monitor_jobs(jobs, input_position_dirpaths) + monitor_jobs(jobs, input_position_dirpaths, doPrint) @click.command() diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py index 3f805c29..1bb6c9a9 100644 --- a/recOrder/cli/jobs_mgmt.py +++ b/recOrder/cli/jobs_mgmt.py @@ -10,12 +10,14 @@ SERVER_PORT = 8089 # Choose an available port JOBS_TIMEOUT = 5 # 5 mins SERVER_uIDsjobIDs = {} # uIDsjobIDs[uid][jid] = job + class JobsManagement(): def __init__(self, *args, **kwargs): self.executor = submitit.AutoExecutor(folder="logs") self.clientsocket = None self.uIDsjobIDs = {} # uIDsjobIDs[uid][jid] = job + self.DATA_QUEUE = [] def checkForJobIDFile(self, jobID, logsPath, extension="out"): @@ -35,7 +37,7 @@ def checkForJobIDFile(self, jobID, logsPath, extension="out"): return "" def setShorterTimeout(self): - self.clientsocket.settimeout(3) + self.clientsocket.settimeout(30) def startClient(self): try: @@ -103,21 +105,37 @@ def stopClient(self): print(exc.args) def checkAllExpJobsCompletion(self, uID): - if uID in self.uIDsjobIDs.keys(): - for jobEntry in self.uIDsjobIDs[uID]: - jobsBool = jobEntry["jID"] - if jobsBool == False: + if uID in SERVER_uIDsjobIDs.keys(): + for jobEntry in SERVER_uIDsjobIDs[uID].keys(): + job:submitit.Job = SERVER_uIDsjobIDs[uID][jobEntry]["job"] + jobBool = SERVER_uIDsjobIDs[uID][jobEntry]["bool"] + if job is not None and job.done() == False: + return False + if jobBool == False: return False return True def putJobCompletionInList(self, jobBool, uID: str, jID: str, mode="client"): - if uID in self.uIDsjobIDs.keys(): - if jID in self.uIDsjobIDs[uID].keys(): - self.uIDsjobIDs[uID][jID] = jobBool + if uID in SERVER_uIDsjobIDs.keys(): + if jID in SERVER_uIDsjobIDs[uID].keys(): + SERVER_uIDsjobIDs[uID][jID]["bool"] = jobBool + + def addData(self, data): + self.DATA_QUEUE.append(data) + + def sendDataThread(self): + thread = threading.Thread(target=self.sendData) + thread.start() + + def sendData(self): + data = "".join(self.DATA_QUEUE) + self.clientsocket.send(data.encode()) + self.DATA_QUEUE = [] def putJobInList(self, job, uID: str, jID: str, well:str, log_folder_path:str="", mode="client"): try: well = str(well) + jID = str(jID) if ".zarr" in well: wells = well.split(".zarr") well = wells[1].replace("\\","-").replace("/","-")[1:] @@ -129,32 +147,26 @@ def putJobInList(self, job, uID: str, jID: str, well:str, log_folder_path:str="" if jID not in self.uIDsjobIDs[uID].keys(): self.uIDsjobIDs[uID][jID] = job json_obj = {uID:{"jID": str(jID), "pos": well, "log": log_folder_path}} - json_str = json.dumps(json_obj)+"\n" - self.clientsocket.send(json_str.encode()) + json_str = json.dumps(json_obj)+"\n" + self.addData(json_str) else: # from server side jobs object entry is a None object # this will be later checked as completion boolean for a ExpID which might # have several Jobs associated with it if uID not in SERVER_uIDsjobIDs.keys(): SERVER_uIDsjobIDs[uID] = {} - SERVER_uIDsjobIDs[uID][jID] = job - else: - if jID not in SERVER_uIDsjobIDs[uID].keys(): - SERVER_uIDsjobIDs[uID][jID] = job + SERVER_uIDsjobIDs[uID][jID] = {} + SERVER_uIDsjobIDs[uID][jID]["job"] = job + SERVER_uIDsjobIDs[uID][jID]["bool"] = False + else: + SERVER_uIDsjobIDs[uID][jID] = {} + SERVER_uIDsjobIDs[uID][jID]["job"] = job + SERVER_uIDsjobIDs[uID][jID]["bool"] = False except Exception as exc: print(exc.args) - - def hasSubmittedJob(self, uID: str, mode="client")->bool: - if mode == "client": - if uID in self.uIDsjobIDs.keys(): - return True - return False - else: - if uID in SERVER_uIDsjobIDs.keys(): - return True - return False def hasSubmittedJob(self, uID: str, jID: str, mode="client")->bool: + jID = str(jID) if mode == "client": if uID in self.uIDsjobIDs.keys(): if jID in self.uIDsjobIDs[uID].keys(): diff --git a/recOrder/cli/monitor.py b/recOrder/cli/monitor.py index a86b7fa6..3526d011 100644 --- a/recOrder/cli/monitor.py +++ b/recOrder/cli/monitor.py @@ -7,28 +7,31 @@ import sys -def _move_cursor_up(n_lines): - sys.stdout.write("\033[F" * n_lines) +def _move_cursor_up(n_lines, doPrint=True): + if doPrint: + sys.stdout.write("\033[F" * n_lines) -def _print_status(jobs, position_dirpaths, elapsed_list, print_indices=None): +def _print_status(jobs, position_dirpaths, elapsed_list, print_indices=None, doPrint=True): columns = [15, 30, 40, 50] # header - sys.stdout.write( - "\033[K" # clear line - "\033[96mID" # cyan - f"\033[{columns[0]}G WELL " - f"\033[{columns[1]}G STATUS " - f"\033[{columns[2]}G NODE " - f"\033[{columns[2]}G ELAPSED\n" - ) + if doPrint: + sys.stdout.write( + "\033[K" # clear line + "\033[96mID" # cyan + f"\033[{columns[0]}G WELL " + f"\033[{columns[1]}G STATUS " + f"\033[{columns[2]}G NODE " + f"\033[{columns[2]}G ELAPSED\n" + ) if print_indices is None: print_indices = range(len(jobs)) complete_count = 0 + for i, (job, position_dirpath) in enumerate(zip(jobs, position_dirpaths)): try: node_name = job.get_info()["NodeList"] # slowest, so do this first @@ -43,22 +46,24 @@ def _print_status(jobs, position_dirpaths, elapsed_list, print_indices=None): elapsed_list[i] += 1 # inexact timing else: color = "\033[91m" # red - + if i in print_indices: - sys.stdout.write( - f"\033[K" # clear line - f"{color}{job.job_id}" - f"\033[{columns[0]}G {'/'.join(position_dirpath.parts[-3:])}" - f"\033[{columns[1]}G {job.state}" - f"\033[{columns[2]}G {node_name}" - f"\033[{columns[3]}G {elapsed_list[i]} s\n" - ) + if doPrint: + sys.stdout.write( + f"\033[K" # clear line + f"{color}{job.job_id}" + f"\033[{columns[0]}G {'/'.join(position_dirpath.parts[-3:])}" + f"\033[{columns[1]}G {job.state}" + f"\033[{columns[2]}G {node_name}" + f"\033[{columns[3]}G {elapsed_list[i]} s\n" + ) sys.stdout.flush() - print( - f"\033[32m{complete_count}/{len(jobs)} jobs complete. " - " to move monitor to background. " - " twice to cancel jobs." - ) + if doPrint: + print( + f"\033[32m{complete_count}/{len(jobs)} jobs complete. " + " to move monitor to background. " + " twice to cancel jobs." + ) return elapsed_list @@ -87,7 +92,7 @@ def _get_jobs_to_print(jobs, num_to_print): return job_indices_to_print -def monitor_jobs(jobs: list[submitit.Job], position_dirpaths: list[Path]): +def monitor_jobs(jobs: list[submitit.Job], position_dirpaths: list[Path], doPrint=True): """Displays the status of a list of submitit jobs with corresponding paths. Parameters @@ -108,7 +113,7 @@ def monitor_jobs(jobs: list[submitit.Job], position_dirpaths: list[Path]): # print all jobs once if terminal is too small if shutil.get_terminal_size().lines - NON_JOB_LINES < len(jobs): - _print_status(jobs, position_dirpaths, elapsed_list) + _print_status(jobs, position_dirpaths, elapsed_list, doPrint) # main monitor loop try: @@ -125,14 +130,15 @@ def monitor_jobs(jobs: list[submitit.Job], position_dirpaths: list[Path]): position_dirpaths, elapsed_list, job_indices_to_print, + doPrint, ) time.sleep(1) - _move_cursor_up(num_jobs_to_print + 2) + _move_cursor_up(num_jobs_to_print + 2, doPrint) # Print final status time.sleep(1) - _print_status(jobs, position_dirpaths, elapsed_list) + _print_status(jobs, position_dirpaths, elapsed_list, doPrint=doPrint) # cancel jobs if ctrl+c except KeyboardInterrupt: diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 4f1ee11d..7e260685 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -1008,7 +1008,7 @@ def add_widget(self, parentLayout:QVBoxLayout, expID, jID, tableEntryID="", pos= tableEntryID + " - " + pos ) # tableEntryID, tableEntryShortDesc - should update with processing status _collapsibleBoxWidget.setSizePolicy( - QSizePolicy.Expanding, QSizePolicy.Fixed + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed ) _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) @@ -1144,7 +1144,7 @@ def addTableEntry( _expandingTabEntryWidget = QWidget() _expandingTabEntryWidget.toolTip = tableEntryShortDesc _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) proc_params["tableEntryID"] = tableEntryID proc_params["parent_layout"] = _scrollAreaCollapsibleBoxWidgetLayout @@ -1377,7 +1377,11 @@ def _create_acq_contols2( ) _show_CheckBox = widgets.CheckBox(name="Show after Reconstruction", value=True) - _validate_button = widgets.PushButton(name="Validate") + _show_CheckBox.max_width = 200 + _rx_Label = widgets.Label(value="rx") + _rx_LineEdit = widgets.LineEdit(name="rx", value=1) + _rx_LineEdit.max_width = 50 + _validate_button = widgets.PushButton(name="Validate") # Passing all UI components that would be deleted _expandingTabEntryWidget = QWidget() @@ -1435,9 +1439,11 @@ def _create_acq_contols2( _hBox_layout2 = QHBoxLayout() _hBox_layout2.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _hBox_widget2.setLayout(_hBox_layout2) - _hBox_layout2.addWidget(_show_CheckBox.native) + _hBox_layout2.addWidget(_show_CheckBox.native) _hBox_layout2.addWidget(_validate_button.native) _hBox_layout2.addWidget(_del_button.native) + _hBox_layout2.addWidget(_rx_Label.native) + _hBox_layout2.addWidget(_rx_LineEdit.native) _expandingTabEntryWidgetLayout = QVBoxLayout() _expandingTabEntryWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) @@ -1480,6 +1486,7 @@ def _create_acq_contols2( "exclude_modes": exclude_modes.copy(), "poll_data": self.pollData, "show": _show_CheckBox, + "rx":_rx_LineEdit, } ) self.index += 1 @@ -1896,6 +1903,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): proc_params["output_path"] = str(Path(output_dir).absolute()) proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) proc_params["show"] = item["show"].value + proc_params["rx"] = item["rx"].value self.addTableEntry( tableID, tableDescToolTip, proc_params @@ -2033,6 +2041,7 @@ def addPollLoop(self, input_data_path, last_time_index): proc_params["output_path"] = str(Path(output_dir).absolute()) proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) proc_params["show"] = False + proc_params["rx"] = 1 tableEntryWorker1 = AddTableEntryWorkerThread(tableID, tableDescToolTip, proc_params) tableEntryWorker1.add_tableentry_signal.connect(self.addTableEntry) @@ -2554,7 +2563,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol jobIdx = str(jobIdx) if client_socket is not None and expIdx=="" and jobIdx=="": try: - buf = client_socket.recv(1024) + buf = client_socket.recv(10240) if len(buf) > 0: if b"\n" in buf: dataList = buf.split(b"\n") @@ -2584,6 +2593,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol proc_params["output_path"] = str(output_data) proc_params["output_path_parent"] = str(Path(output_data).parent.absolute()) proc_params["show"] = False + proc_params["rx"] = 1 if config_path == "": model = None @@ -2688,11 +2698,12 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms expIdx = expIdx +"-"+ unique_id self.JobsMgmt.putJobInList(None, expIdx, str(jobIdx), wellName, mode="server") + # print("Submitting Job: {job} expIdx: {expIdx}".format(job=jobIdx, expIdx=expIdx)) thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=(expIdx, jobIdx, wellName, logs_folder_path, client_socket)) thread.start() return - except: - pass + except Exception as exc: + print(exc.args) # ToDo: Another approach to this could be to implement a status thread on the client side # Since the client is already running till the job is completed, the client could ping status @@ -2716,6 +2727,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol proc_params["output_path"] = "" proc_params["output_path_parent"] = "" proc_params["show"] = False + proc_params["rx"] = 1 tableEntryWorker = AddTableEntryWorkerThread(tableID, tableID, proc_params) tableEntryWorker.add_tableentry_signal.connect(self.tab_recon.addTableEntry) @@ -2779,8 +2791,9 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol ) # 5 mins - match executor time-out _lastUpdate_jobTXT = "" jobTXT = "" + # print("Updating Job: {job} expIdx: {expIdx}".format(job=jobIdx, expIdx=expIdx)) while True: - time.sleep(1) # update every sec and exit on break + time.sleep(1) # update every sec and exit on break try: if "cancel called" in _cancelJobBtn.text: json_obj = {"uID":expIdx, "jID":jobIdx, "command":"cancel"} @@ -2792,22 +2805,22 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + "Please check terminal output for Job status..\n\n" + jobTXT ) - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=1) break # cancel called by user if _infoBox == None: params["status"] = STATUS_user_cleared_job - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=2) break # deleted by user - no longer needs updating if _infoBox: pass except Exception as exc: print(exc.args) params["status"] = STATUS_user_cleared_job - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=3) break # deleted by user - no longer needs updating if self.JobsMgmt.hasSubmittedJob(expIdx, jobIdx, mode="server"): if params["status"] in [STATUS_finished_job]: - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=4) break elif params["status"] in [STATUS_errored_job]: jobERR = self.JobsMgmt.checkForJobIDFile( @@ -2816,7 +2829,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol _infoBox.setText( jobIdx + "\n" + params["desc"] + "\n\n" + jobERR ) - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=5) break else: jobTXT = self.JobsMgmt.checkForJobIDFile( @@ -2824,6 +2837,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol ) try: if jobTXT == "": # job file not created yet + # print(jobIdx + " not started yet") time.sleep(2) _tUpdateCount += 2 if _tUpdateCount > 10: # if out file is empty for 10s, check the err file to update user @@ -2838,7 +2852,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + jobERR ) if _tUpdateCount > _tUpdateCountTimeout: - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=6) break elif ( params["status"] @@ -2848,10 +2862,10 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol # check to ensure row deletion due to shrinking table # if not deleted try to delete again if rowIdx < 0: - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=7) break else: - ROW_POP_QUEUE.append(expIdx) + break elif JOB_COMPLETION_STR in jobTXT: params["status"] = STATUS_finished_job _infoBox.setText(jobTXT) @@ -2859,7 +2873,6 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol # we cant delete the row directly from this thread # we will use the exp_id to identify and delete the row # using pyqtSignal - ROW_POP_QUEUE.append(expIdx) # break - based on status elif JOB_TRIGGERED_EXC in jobTXT: params["status"] = STATUS_errored_job @@ -2875,7 +2888,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + "\n\n" + jobERR ) - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=8) break elif JOB_RUNNING_STR in jobTXT: params["status"] = STATUS_running_job @@ -2892,7 +2905,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + jobTXT ) if _tUpdateCount > _tUpdateCountTimeout: - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=9) break else: jobERR = self.JobsMgmt.checkForJobIDFile( @@ -2905,12 +2918,12 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + "\n\n" + jobERR ) - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=10) break except Exception as exc: print(exc.args) else: - self.clientRelease(expIdx, jobIdx, client_socket, params) + self.clientRelease(expIdx, jobIdx, client_socket, params, reason=11) break else: # this would occur when an exception happens on the pool side before or during job submission @@ -2924,8 +2937,9 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol poolERR = params["error"] _infoBox.setText(poolERR) - def clientRelease(self, expIdx, jobIdx, client_socket, params): + def clientRelease(self, expIdx, jobIdx, client_socket, params, reason=0): # only need to release client from primary job + # print("clientRelease Job: {job} expIdx: {expIdx} reason:{reason}".format(job=jobIdx, expIdx=expIdx, reason=reason)) self.JobsMgmt.putJobCompletionInList(True, expIdx, jobIdx) showData_thread = None if params["primary"]: @@ -2938,9 +2952,12 @@ def clientRelease(self, expIdx, jobIdx, client_socket, params): while not self.JobsMgmt.checkAllExpJobsCompletion(expIdx): time.sleep(1) + json_obj = {"uID":expIdx, "jID":jobIdx,"command": "clientRelease"} json_str = json.dumps(json_obj)+"\n" client_socket.send(json_str.encode()) + ROW_POP_QUEUE.append(expIdx) + # print("FINISHED") if self.pool is not None: if self.pool._work_queue.qsize() == 0: @@ -3024,6 +3041,7 @@ def runInSubProcess(self, params): config_path = str(params["config_path"]) output_path = str(params["output_path"]) uid = str(params["exp_id"]) + rx = str(params["rx"]) mainfp = str(jobs_mgmt.FILE_PATH) self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_submitted_job @@ -3040,7 +3058,7 @@ def runInSubProcess(self, params): "-o", output_path, "-rx", - str(20), + str(rx), "-uid", uid ] @@ -3203,12 +3221,22 @@ def __init__(self, text, *args, **kwargs): layout = QVBoxLayout() layout.setAlignment(Qt.AlignmentFlag.AlignTop) layout.addWidget(self.label) + self.label.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) container = QWidget() container.setLayout(layout) + container.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) self.setWidget(container) self.setWidgetResizable(True) + self.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) + self.setAlignment(Qt.AlignmentFlag.AlignTop) def setText(self, text): self.label.setText(text) From a80a337dc92c8b80fc07119df1f3d21165043973 Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Tue, 21 Jan 2025 12:07:07 -0500 Subject: [PATCH 37/38] code formatting, minor refactoring, comments --- .../cli/apply_inverse_transfer_function.py | 8 +- recOrder/cli/jobs_mgmt.py | 49 +- recOrder/cli/monitor.py | 22 +- recOrder/plugin/main_widget.py | 2 +- recOrder/plugin/tab_recon.py | 1694 +++++++++++------ recOrder/scripts/simulate_zarr_acq.py | 53 +- 6 files changed, 1204 insertions(+), 624 deletions(-) diff --git a/recOrder/cli/apply_inverse_transfer_function.py b/recOrder/cli/apply_inverse_transfer_function.py index c4172735..afb31e01 100644 --- a/recOrder/cli/apply_inverse_transfer_function.py +++ b/recOrder/cli/apply_inverse_transfer_function.py @@ -377,16 +377,16 @@ def apply_inverse_transfer_function_cli( doPrint = True # CLI prints Job status when used as cmd line if unique_id != "": # no unique_id means no job submission info being listened to - JM.startClient() + JM.start_client() i=0 for j in jobs: job : submitit.Job = j job_idx : str = job.job_id position = input_position_dirpaths[i] - JM.putJobInList(job, unique_id, str(job_idx), position, str(executor.folder.absolute())) + JM.put_Job_in_list(job, unique_id, str(job_idx), position, str(executor.folder.absolute())) i += 1 - JM.sendDataThread() - JM.setShorterTimeout() + JM.send_data_thread() + JM.set_shorter_timeout() doPrint = False # CLI printing disabled when using GUI monitor_jobs(jobs, input_position_dirpaths, doPrint) diff --git a/recOrder/cli/jobs_mgmt.py b/recOrder/cli/jobs_mgmt.py index 1bb6c9a9..4e833631 100644 --- a/recOrder/cli/jobs_mgmt.py +++ b/recOrder/cli/jobs_mgmt.py @@ -14,20 +14,19 @@ class JobsManagement(): def __init__(self, *args, **kwargs): - self.executor = submitit.AutoExecutor(folder="logs") self.clientsocket = None self.uIDsjobIDs = {} # uIDsjobIDs[uid][jid] = job self.DATA_QUEUE = [] - def checkForJobIDFile(self, jobID, logsPath, extension="out"): + def check_for_jobID_File(self, jobID, logs_path, extension="out"): - if Path(logsPath).exists(): - files = os.listdir(logsPath) + if Path(logs_path).exists(): + files = os.listdir(logs_path) try: for file in files: if file.endswith(extension): if jobID in file: - file_path = os.path.join(logsPath, file) + file_path = os.path.join(logs_path, file) f = open(file_path, "r") txt = f.read() f.close() @@ -36,22 +35,28 @@ def checkForJobIDFile(self, jobID, logsPath, extension="out"): print(exc.args) return "" - def setShorterTimeout(self): + def set_shorter_timeout(self): self.clientsocket.settimeout(30) - - def startClient(self): + + def start_client(self): try: self.clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.clientsocket.settimeout(300) self.clientsocket.connect(('localhost', SERVER_PORT)) self.clientsocket.settimeout(None) - thread = threading.Thread(target=self.stopClient) + thread = threading.Thread(target=self.stop_client) thread.start() except Exception as exc: print(exc.args) - def stopClient(self): + # The stopClient() is called right with the startClient() but does not stop + # and essentially is a wait thread listening and is triggered by either a + # connection or timeout. Based on condition triggered by user, reconstruction + # completion or errors the end goal is to close the socket connection which + # would let the CLI exit. I could break it down to 2 parts but the idea was to + # keep the clientsocket.close() call within one method to make it easier to follow. + def stop_client(self): try: time.sleep(2) while True: @@ -73,11 +78,11 @@ def stopClient(self): job_idx = str(json_obj["jID"]) cmd = json_obj["command"] if cmd == "clientRelease": - if self.hasSubmittedJob(u_idx, job_idx): + if self.has_submitted_job(u_idx, job_idx): self.clientsocket.close() break if cmd == "cancel": - if self.hasSubmittedJob(u_idx, job_idx): + if self.has_submitted_job(u_idx, job_idx): try: job = self.uIDsjobIDs[u_idx][job_idx] job.cancel() @@ -104,7 +109,7 @@ def stopClient(self): self.clientsocket.close() print(exc.args) - def checkAllExpJobsCompletion(self, uID): + def check_all_ExpJobs_completion(self, uID): if uID in SERVER_uIDsjobIDs.keys(): for jobEntry in SERVER_uIDsjobIDs[uID].keys(): job:submitit.Job = SERVER_uIDsjobIDs[uID][jobEntry]["job"] @@ -115,24 +120,24 @@ def checkAllExpJobsCompletion(self, uID): return False return True - def putJobCompletionInList(self, jobBool, uID: str, jID: str, mode="client"): + def put_Job_completion_in_list(self, job_bool, uID: str, jID: str, mode="client"): if uID in SERVER_uIDsjobIDs.keys(): if jID in SERVER_uIDsjobIDs[uID].keys(): - SERVER_uIDsjobIDs[uID][jID]["bool"] = jobBool + SERVER_uIDsjobIDs[uID][jID]["bool"] = job_bool - def addData(self, data): + def add_data(self, data): self.DATA_QUEUE.append(data) - def sendDataThread(self): - thread = threading.Thread(target=self.sendData) + def send_data_thread(self): + thread = threading.Thread(target=self.send_data) thread.start() - def sendData(self): + def send_data(self): data = "".join(self.DATA_QUEUE) self.clientsocket.send(data.encode()) self.DATA_QUEUE = [] - def putJobInList(self, job, uID: str, jID: str, well:str, log_folder_path:str="", mode="client"): + def put_Job_in_list(self, job, uID: str, jID: str, well:str, log_folder_path:str="", mode="client"): try: well = str(well) jID = str(jID) @@ -148,7 +153,7 @@ def putJobInList(self, job, uID: str, jID: str, well:str, log_folder_path:str="" self.uIDsjobIDs[uID][jID] = job json_obj = {uID:{"jID": str(jID), "pos": well, "log": log_folder_path}} json_str = json.dumps(json_obj)+"\n" - self.addData(json_str) + self.add_data(json_str) else: # from server side jobs object entry is a None object # this will be later checked as completion boolean for a ExpID which might @@ -165,7 +170,7 @@ def putJobInList(self, job, uID: str, jID: str, well:str, log_folder_path:str="" except Exception as exc: print(exc.args) - def hasSubmittedJob(self, uID: str, jID: str, mode="client")->bool: + def has_submitted_job(self, uID: str, jID: str, mode="client")->bool: jID = str(jID) if mode == "client": if uID in self.uIDsjobIDs.keys(): diff --git a/recOrder/cli/monitor.py b/recOrder/cli/monitor.py index 3526d011..474637af 100644 --- a/recOrder/cli/monitor.py +++ b/recOrder/cli/monitor.py @@ -7,17 +7,17 @@ import sys -def _move_cursor_up(n_lines, doPrint=True): - if doPrint: +def _move_cursor_up(n_lines, do_print=True): + if do_print: sys.stdout.write("\033[F" * n_lines) -def _print_status(jobs, position_dirpaths, elapsed_list, print_indices=None, doPrint=True): +def _print_status(jobs, position_dirpaths, elapsed_list, print_indices=None, do_print=True): columns = [15, 30, 40, 50] # header - if doPrint: + if do_print: sys.stdout.write( "\033[K" # clear line "\033[96mID" # cyan @@ -48,7 +48,7 @@ def _print_status(jobs, position_dirpaths, elapsed_list, print_indices=None, doP color = "\033[91m" # red if i in print_indices: - if doPrint: + if do_print: sys.stdout.write( f"\033[K" # clear line f"{color}{job.job_id}" @@ -58,7 +58,7 @@ def _print_status(jobs, position_dirpaths, elapsed_list, print_indices=None, doP f"\033[{columns[3]}G {elapsed_list[i]} s\n" ) sys.stdout.flush() - if doPrint: + if do_print: print( f"\033[32m{complete_count}/{len(jobs)} jobs complete. " " to move monitor to background. " @@ -92,7 +92,7 @@ def _get_jobs_to_print(jobs, num_to_print): return job_indices_to_print -def monitor_jobs(jobs: list[submitit.Job], position_dirpaths: list[Path], doPrint=True): +def monitor_jobs(jobs: list[submitit.Job], position_dirpaths: list[Path], do_print=True): """Displays the status of a list of submitit jobs with corresponding paths. Parameters @@ -113,7 +113,7 @@ def monitor_jobs(jobs: list[submitit.Job], position_dirpaths: list[Path], doPrin # print all jobs once if terminal is too small if shutil.get_terminal_size().lines - NON_JOB_LINES < len(jobs): - _print_status(jobs, position_dirpaths, elapsed_list, doPrint) + _print_status(jobs, position_dirpaths, elapsed_list, do_print) # main monitor loop try: @@ -130,15 +130,15 @@ def monitor_jobs(jobs: list[submitit.Job], position_dirpaths: list[Path], doPrin position_dirpaths, elapsed_list, job_indices_to_print, - doPrint, + do_print, ) time.sleep(1) - _move_cursor_up(num_jobs_to_print + 2, doPrint) + _move_cursor_up(num_jobs_to_print + 2, do_print) # Print final status time.sleep(1) - _print_status(jobs, position_dirpaths, elapsed_list, doPrint=doPrint) + _print_status(jobs, position_dirpaths, elapsed_list, do_print=do_print) # cancel jobs if ctrl+c except KeyboardInterrupt: diff --git a/recOrder/plugin/main_widget.py b/recOrder/plugin/main_widget.py index 947ac537..1f2b5ff9 100644 --- a/recOrder/plugin/main_widget.py +++ b/recOrder/plugin/main_widget.py @@ -90,7 +90,7 @@ def __init__(self, napari_viewer: Viewer): # Setup GUI elements self.ui = gui.Ui_Form() self.ui.setupUi(self) - self.ui.tab_reconstruction.setViewer(napari_viewer) + self.ui.tab_reconstruction.set_viewer(napari_viewer) # Override initial tab focus self.ui.tabWidget.setCurrentIndex(0) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index 7e260685..fafec751 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -46,11 +46,11 @@ "Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format( v=version ) - ) + ) from pydantic.main import ValidationError from pydantic.main import BaseModel from pydantic.main import ModelMetaclass - elif version >= "1.10.19": + elif version >= "1.10.19": from pydantic.main import ValidationError from pydantic.main import BaseModel from pydantic.main import ModelMetaclass @@ -59,7 +59,7 @@ "Your Pydantic library ver:{v}. Recommended ver is: 1.10.19".format( v=version ) - ) + ) from pydantic.main import ValidationError from pydantic.main import BaseModel from pydantic.main import ModelMetaclass @@ -106,6 +106,7 @@ NEW_WIDGETS_QUEUE = [] NEW_WIDGETS_QUEUE_THREADS = [] MULTI_JOBS_REFS = {} +ROW_POP_QUEUE = [] # Main class for the Reconstruction tab # Not efficient since instantiated from GUI @@ -117,7 +118,7 @@ def __init__(self, parent=None, stand_alone=False): super().__init__(parent) self._ui = parent self.stand_alone = stand_alone - self.viewer:Viewer = None + self.viewer: Viewer = None if HAS_INSTANCE["val"]: self.current_dir_path = str(Path.cwd()) self.directory = str(Path.cwd()) @@ -142,7 +143,9 @@ def __init__(self, parent=None, stand_alone=False): self.recon_tab_mainScrollArea.setWidgetResizable(True) self.recon_tab_widget = QWidget() - self.recon_tab_widget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + self.recon_tab_widget.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) self.recon_tab_layout = QVBoxLayout() self.recon_tab_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) self.recon_tab_layout.setContentsMargins(0, 0, 0, 0) @@ -160,7 +163,9 @@ def __init__(self, parent=None, stand_alone=False): # Input Data ############################## self.data_input_widget = QWidget() self.data_input_widget_layout = QHBoxLayout() - self.data_input_widget_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.data_input_widget_layout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) self.data_input_widget.setLayout(self.data_input_widget_layout) self.data_input_Label = widgets.Label(value="Input Store") @@ -169,28 +174,46 @@ def __init__(self, parent=None, stand_alone=False): self.data_input_PushButton = widgets.PushButton(label="Browse") # self.data_input_PushButton.native.setMinimumWidth(75) self.data_input_PushButton.clicked.connect(self.browse_dir_path_input) - self.data_input_LineEdit.changed.connect(self.readAndSetInputPathOnValidation) - + self.data_input_LineEdit.changed.connect( + self.read_and_set_input_path_on_validation + ) + self.data_input_widget_layout.addWidget(self.data_input_Label.native) - self.data_input_widget_layout.addWidget(self.data_input_LineEdit.native) - self.data_input_widget_layout.addWidget(self.data_input_PushButton.native) + self.data_input_widget_layout.addWidget( + self.data_input_LineEdit.native + ) + self.data_input_widget_layout.addWidget( + self.data_input_PushButton.native + ) # Output Data ############################## self.data_output_widget = QWidget() self.data_output_widget_layout = QHBoxLayout() - self.data_output_widget_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.data_output_widget_layout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) self.data_output_widget.setLayout(self.data_output_widget_layout) self.data_output_Label = widgets.Label(value="Output Directory") - self.data_output_LineEdit = widgets.LineEdit(value=self.output_directory) + self.data_output_LineEdit = widgets.LineEdit( + value=self.output_directory + ) self.data_output_PushButton = widgets.PushButton(label="Browse") # self.data_output_PushButton.native.setMinimumWidth(75) - self.data_output_PushButton.clicked.connect(self.browse_dir_path_output) - self.data_output_LineEdit.changed.connect(self.readAndSetOutPathOnValidation) - + self.data_output_PushButton.clicked.connect( + self.browse_dir_path_output + ) + self.data_output_LineEdit.changed.connect( + self.read_and_set_out_path_on_validation + ) + self.data_output_widget_layout.addWidget(self.data_output_Label.native) - self.data_output_widget_layout.addWidget(self.data_output_LineEdit.native) - self.data_output_widget_layout.addWidget(self.data_output_PushButton.native) + self.data_output_widget_layout.addWidget( + self.data_output_LineEdit.native + ) + self.data_output_widget_layout.addWidget( + self.data_output_PushButton.native + ) self.data_input_Label.native.setMinimumWidth(115) self.data_output_Label.native.setMinimumWidth(115) @@ -213,7 +236,9 @@ def __init__(self, parent=None, stand_alone=False): self.models_widget = QWidget() self.models_widget_layout = QHBoxLayout() - self.models_widget_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.models_widget_layout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) self.models_widget.setLayout(self.models_widget_layout) self.modes_selected = OPTION_TO_MODEL_DICT.copy() @@ -232,32 +257,29 @@ def __init__(self, parent=None, stand_alone=False): ) # PushButton to create a copy of the model - UI - self.models_new_PushButton = widgets.PushButton( - label="New" - ) + self.models_new_PushButton = widgets.PushButton(label="New") # self.models_new_PushButton.native.setMinimumWidth(100) - self.models_new_PushButton.clicked.connect( - self._build_acq_contols - ) + self.models_new_PushButton.clicked.connect(self.build_acq_contols) self.models_load_PushButton = DropButton(text="Load", recon_tab=self) # self.models_load_PushButton.setMinimumWidth(90) - + # Passing model location label to model location selector self.models_load_PushButton.clicked.connect( - lambda: self.browse_dir_path_model()) + lambda: self.browse_dir_path_model() + ) # PushButton to clear all copies of models that are create for UI - self.models_clear_PushButton = widgets.PushButton( - label="Clear" - ) + self.models_clear_PushButton = widgets.PushButton(label="Clear") # self.models_clear_PushButton.native.setMinimumWidth(110) - self.models_clear_PushButton.clicked.connect(self._clear_all_models) + self.models_clear_PushButton.clicked.connect(self.clear_all_models) - self.models_widget_layout.addWidget(self.models_new_PushButton.native) + self.models_widget_layout.addWidget(self.models_new_PushButton.native) self.models_widget_layout.addWidget(self.models_load_PushButton) - self.models_widget_layout.addWidget(self.models_clear_PushButton.native) - + self.models_widget_layout.addWidget( + self.models_clear_PushButton.native + ) + # Middle scrollable component which will hold Editable/(vertical) Expanding UI self.models_scrollArea = QScrollArea() self.models_scrollArea.setWidgetResizable(True) @@ -265,10 +287,14 @@ def __init__(self, parent=None, stand_alone=False): self.models_container_widget_layout = QVBoxLayout() self.models_container_widget_layout.setContentsMargins(0, 0, 0, 0) self.models_container_widget_layout.setSpacing(2) - self.models_container_widget_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - self.models_container_widget.setLayout(self.models_container_widget_layout) + self.models_container_widget_layout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) + self.models_container_widget.setLayout( + self.models_container_widget_layout + ) self.models_scrollArea.setWidget(self.models_container_widget) - + group_box_Models_layout.addWidget(self.models_widget) group_box_Models_layout.addWidget(self.models_scrollArea) @@ -278,30 +304,42 @@ def __init__(self, parent=None, stand_alone=False): splitter = QSplitter() splitter.setOrientation(Qt.Orientation.Vertical) splitter.setSizes([600, 200]) - + self.recon_tab_layout.addWidget(splitter) - + # Reconstruction ################################## # Run, Processing, On-The-Fly - group_box_Reconstruction_groupBox_widget = QGroupBox("Reconstruction Queue") + group_box_Reconstruction_groupBox_widget = QGroupBox( + "Reconstruction Queue" + ) group_box_Reconstruction_layout = QVBoxLayout() group_box_Reconstruction_layout.setContentsMargins(5, 10, 5, 5) group_box_Reconstruction_layout.setSpacing(2) - group_box_Reconstruction_groupBox_widget.setLayout(group_box_Reconstruction_layout) - + group_box_Reconstruction_groupBox_widget.setLayout( + group_box_Reconstruction_layout + ) + splitter.addWidget(group_box_Models_groupBox_widget) splitter.addWidget(group_box_Reconstruction_groupBox_widget) - my_splitter_handle = splitter.handle(1) + my_splitter_handle = splitter.handle(1) my_splitter_handle.setStyleSheet("background: 1px rgb(128,128,128);") - splitter.setStyleSheet("""QSplitter::handle:pressed {background-color: #ca5;}""") + splitter.setStyleSheet( + """QSplitter::handle:pressed {background-color: #ca5;}""" + ) # PushButton to validate and Run the yaml file(s) based on selection against the Input store - self.reconstruction_run_PushButton = widgets.PushButton(name="RUN Model") + self.reconstruction_run_PushButton = widgets.PushButton( + name="RUN Model" + ) self.reconstruction_run_PushButton.native.setMinimumWidth(100) - self.reconstruction_run_PushButton.clicked.connect(self.build_model_and_run) + self.reconstruction_run_PushButton.clicked.connect( + self.build_model_and_run + ) - group_box_Reconstruction_layout.addWidget(self.reconstruction_run_PushButton.native) + group_box_Reconstruction_layout.addWidget( + self.reconstruction_run_PushButton.native + ) # Tabs - Processing & On-The-Fly tabs_Reconstruction = QTabWidget() @@ -314,9 +352,13 @@ def __init__(self, parent=None, stand_alone=False): tab1_processing_widget_layout.setSpacing(2) tab1_processing_widget.setLayout(tab1_processing_widget_layout) self.proc_table_QFormLayout = QFormLayout() - self.proc_table_QFormLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.proc_table_QFormLayout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) tab1_processing_form_widget = QWidget() - tab1_processing_form_widget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding) + tab1_processing_form_widget.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding + ) tab1_processing_form_widget.setLayout(self.proc_table_QFormLayout) tab1_processing_widget_layout.addWidget(tab1_processing_form_widget) @@ -331,15 +373,19 @@ def __init__(self, parent=None, stand_alone=False): tab2_processing_widget_layout.setSpacing(0) tab2_processing_widget.setLayout(tab2_processing_widget_layout) self.proc_OTF_table_QFormLayout = QFormLayout() - self.proc_OTF_table_QFormLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + self.proc_OTF_table_QFormLayout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) _proc_OTF_table_widget = QWidget() - _proc_OTF_table_widget.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) - _proc_OTF_table_widget.setLayout(self.proc_OTF_table_QFormLayout) + _proc_OTF_table_widget.setSizePolicy( + QSizePolicy.Expanding, QSizePolicy.Expanding + ) + _proc_OTF_table_widget.setLayout(self.proc_OTF_table_QFormLayout) tab2_processing_widget_layout.addWidget(_proc_OTF_table_widget) tab2_processing_widget.setMaximumHeight(100) tabs_Reconstruction.addTab(tab1_processing_widget, "Processing") - tabs_Reconstruction.addTab(tab2_processing_widget, "On-The-Fly") + tabs_Reconstruction.addTab(tab2_processing_widget, "On-The-Fly") # Editable List holding pydantic class(es) as per user selection self.pydantic_classes = list() @@ -349,11 +395,11 @@ def __init__(self, parent=None, stand_alone=False): # Stores Model & Components values which cause validation failure - can be highlighted on the model field as Red self.modelHighlighterVals = {} - + # handle napari's close widget and avoid starting a second server if HAS_INSTANCE["val"]: self.worker: MyWorker = HAS_INSTANCE["MyWorker"] - self.worker.setNewInstances( + self.worker.set_new_instances( self.proc_table_QFormLayout, self, self._ui ) else: @@ -377,7 +423,7 @@ def myCloseEvent(self): # on napari close - cleanup def closeEvent(self, event): if event.type() == QEvent.Type.Close: - self.worker.stopServer() + self.worker.stop_server() def hideEvent(self, event): if event.type() == QEvent.Type.Hide and ( @@ -389,18 +435,18 @@ def showEvent(self, event): if event.type() == QEvent.Type.Show: pass - def setViewer(self, viewer): + def set_viewer(self, viewer): self.viewer = viewer - def showDataset(self, data_path): + def show_dataset(self, data_path): # Show reconstruction data try: if self.viewer is not None: self.viewer.open(data_path, plugin="napari-ome-zarr") except Exception as exc: - self.messageBox(exc.args) + self.message_box(exc.args) - def confirmDialog(self, msg="Confirm your selection ?"): + def confirm_dialog(self, msg="Confirm your selection ?"): qm = QMessageBox ret = qm.question( self.recon_tab_widget, @@ -417,78 +463,91 @@ def confirmDialog(self, msg="Confirm your selection ?"): # ToDo: utilize common functions # Input data selector def browse_dir_path_input(self): - if len(self.pydantic_classes)>0 and not self.confirmDialog("Changing Input Data will reset your models. Continue ?"): + if len(self.pydantic_classes) > 0 and not self.confirm_dialog( + "Changing Input Data will reset your models. Continue ?" + ): return else: - self._clear_all_models(silent=True) + self.clear_all_models(silent=True) try: - result = self._open_file_dialog(self.input_directory, "dir", filter="ZARR Storage (*.zarr)") + result = self.open_file_dialog( + self.input_directory, "dir", filter="ZARR Storage (*.zarr)" + ) # .zarr is a folder but we could implement a filter to scan for "ending with" and present those if required except Exception as exc: - self.messageBox(exc.args) + self.message_box(exc.args) return if result == "": return - self.data_input_LineEdit.value = result + self.data_input_LineEdit.value = result - def browse_dir_path_output(self): + def browse_dir_path_output(self): try: - result = self._open_file_dialog(self.output_directory, "dir") + result = self.open_file_dialog(self.output_directory, "dir") except Exception as exc: - self.messageBox(exc.args) + self.message_box(exc.args) return if result == "": return - + if not Path(result).exists(): - self.messageBox("Output Directory path must exist !") - return + self.message_box("Output Directory path must exist !") + return - self.data_output_LineEdit.value = result + self.data_output_LineEdit.value = result def browse_dir_path_inputBG(self, elem): - result = self._open_file_dialog(self.directory, "dir") + result = self.open_file_dialog(self.directory, "dir") if result == "": return - ret, ret_msg = self.validateInputData(result, BG=True) + ret, ret_msg = self.validate_input_data(result, BG=True) if not ret: - self.messageBox(ret_msg) + self.message_box(ret_msg) return elem.value = result - # not working - not used - def validateInputData( + def validate_input_data( self, input_data_folder: str, metadata=False, BG=False ) -> bool: - # Sort and validate the input paths, expanding plates into lists of positions - # return True, MSG_SUCCESS try: self.input_channel_names = [] self.data_input_Label.value = "Input Store" input_paths = Path(input_data_folder) with open_ome_zarr(input_paths, mode="r") as dataset: - # ToDo: Metadata reading and implementation in GUI for - # channel names, time indicies, etc. try: self.input_channel_names = dataset.channel_names - self.data_input_Label.value = "Input Store" + " " + _info_icon - self.data_input_Label.tooltip = "Channel Names:\n- " + "\n- ".join(self.input_channel_names) + self.data_input_Label.value = ( + "Input Store" + " " + _info_icon + ) + self.data_input_Label.tooltip = ( + "Channel Names:\n- " + + "\n- ".join(self.input_channel_names) + ) except Exception as exc: print(exc.args) - + try: - for path, pos in dataset.positions(): + for _, pos in dataset.positions(): axes = pos.zgroup.attrs["multiscales"][0]["axes"] string_array_n = [str(x["name"]) for x in axes] - string_array = [str(x) for x in pos.zgroup.attrs["multiscales"][0]["datasets"][0]["coordinateTransformations"][0]["scale"]] + string_array = [ + str(x) + for x in pos.zgroup.attrs["multiscales"][0][ + "datasets" + ][0]["coordinateTransformations"][0]["scale"] + ] string_scale = [] for i in range(len(string_array_n)): - string_scale.append("{n}={d}".format(n=string_array_n[i], d=string_array[i])) + string_scale.append( + "{n}={d}".format( + n=string_array_n[i], d=string_array[i] + ) + ) txt = "\n\nScale: " + ", ".join(string_scale) self.data_input_Label.tooltip += txt break @@ -501,10 +560,12 @@ def validateInputData( if not BG: self.pollData = False zattrs = dataset.zattrs - if self.isDatasetAcqRunning(zattrs): - if self.confirmDialog(msg="This seems like an in-process Acquisition. Would you like to process data on-the-fly ?"): + if self.is_dataset_acq_running(zattrs): + if self.confirm_dialog( + msg="This seems like an in-process Acquisition. Would you like to process data on-the-fly ?" + ): self.pollData = True - + return True, MSG_SUCCESS raise Exception( "Dataset does not appear to be a valid ome-zarr storage" @@ -514,122 +575,161 @@ def validateInputData( # call back for input LineEdit path changed manually # include data validation - def readAndSetInputPathOnValidation(self): + def read_and_set_input_path_on_validation(self): if ( self.data_input_LineEdit.value is None or len(self.data_input_LineEdit.value) == 0 ): self.data_input_LineEdit.value = self.input_directory - self.messageBox("Input data path cannot be empty") + self.message_box("Input data path cannot be empty") return if not Path(self.data_input_LineEdit.value).exists(): self.data_input_LineEdit.value = self.input_directory - self.messageBox("Input data path must point to a valid location") + self.message_box("Input data path must point to a valid location") return result = self.data_input_LineEdit.value - valid, ret_msg = self.validateInputData(result) + valid, ret_msg = self.validate_input_data(result) - if valid: + if valid: self.directory = Path(result).parent.absolute() self.current_dir_path = result - self.input_directory = result + self.input_directory = result self.prev_model_settings = {} - self.saveLastPaths() + self.save_last_paths() else: self.data_input_LineEdit.value = self.input_directory - self.messageBox(ret_msg) + self.message_box(ret_msg) - self.data_output_LineEdit.value = Path(self.input_directory).parent.absolute() + self.data_output_LineEdit.value = Path( + self.input_directory + ).parent.absolute() - def readAndSetOutPathOnValidation(self): + def read_and_set_out_path_on_validation(self): if ( self.data_output_LineEdit.value is None or len(self.data_output_LineEdit.value) == 0 ): self.data_output_LineEdit.value = self.output_directory - self.messageBox("Output data path cannot be empty") + self.message_box("Output data path cannot be empty") return if not Path(self.data_output_LineEdit.value).exists(): self.data_output_LineEdit.value = self.output_directory - self.messageBox("Output data path must point to a valid location") + self.message_box("Output data path must point to a valid location") return self.output_directory = self.data_output_LineEdit.value - self.validateModelOutputPaths() + self.validate_model_output_paths() - def validateModelOutputPaths(self): + def validate_model_output_paths(self): if len(self.pydantic_classes) > 0: for model_item in self.pydantic_classes: output_LineEdit = model_item["output_LineEdit"] output_Button = model_item["output_Button"] model_item["output_parent_dir"] = self.output_directory - full_out_path = os.path.join(Path(self.output_directory).absolute(), output_LineEdit.value) + full_out_path = os.path.join( + Path(self.output_directory).absolute(), + output_LineEdit.value, + ) model_item["output"] = full_out_path - save_path_exists = True if Path(full_out_path).exists() else False - output_LineEdit.label = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:" - output_LineEdit.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") - output_Button.text = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:" - output_Button.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") + save_path_exists = ( + True if Path(full_out_path).exists() else False + ) + output_LineEdit.label = ( + "" if not save_path_exists else (_validate_alert + " ") + ) + "Output Data:" + output_LineEdit.tooltip = ( + "" + if not save_path_exists + else (_validate_alert + "Output file exists") + ) + output_Button.text = ( + "" if not save_path_exists else (_validate_alert + " ") + ) + "Output Data:" + output_Button.tooltip = ( + "" + if not save_path_exists + else (_validate_alert + "Output file exists") + ) - def isDatasetAcqRunning(self, zattrs: dict)->bool: + def is_dataset_acq_running(self, zattrs: dict) -> bool: """ Checks the zattrs for CurrentDimensions & FinalDimensions key and tries to figure if data acquisition is running """ - required_order = ['time', 'position', 'z', 'channel'] + required_order = ["time", "position", "z", "channel"] if "CurrentDimensions" in zattrs.keys(): my_dict = zattrs["CurrentDimensions"] - sorted_dict_acq = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} + sorted_dict_acq = { + k: my_dict[k] + for k in sorted(my_dict, key=lambda x: required_order.index(x)) + } if "FinalDimensions" in zattrs.keys(): my_dict = zattrs["FinalDimensions"] - sorted_dict_final = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} + sorted_dict_final = { + k: my_dict[k] + for k in sorted(my_dict, key=lambda x: required_order.index(x)) + } if sorted_dict_acq != sorted_dict_final: return True return False - # Copied from main_widget - # ToDo: utilize common functions # Output data selector def browse_model_dir_path_output(self, elem): - result = self._open_file_dialog(self.output_directory, "save") + result = self.open_file_dialog(self.output_directory, "save") if result == "": return - + save_path_exists = True if Path(result).exists() else False - elem.label = "Output Data:" + ("" if not save_path_exists else (" "+_validate_alert)) - elem.tooltip ="" if not save_path_exists else "Output file exists" - + elem.label = "Output Data:" + ( + "" if not save_path_exists else (" " + _validate_alert) + ) + elem.tooltip = "" if not save_path_exists else "Output file exists" + elem.value = Path(result).name - self.saveLastPaths() + self.save_last_paths() # call back for output LineEdit path changed manually - def readAndSetOutputPathOnValidation(self, elem1, elem2, save_path): + def read_and_set_output_path_on_validation(self, elem1, elem2, save_path): if elem1.value is None or len(elem1.value) == 0: elem1.value = Path(save_path).name - - save_path = os.path.join(Path(self.output_directory).absolute(), elem1.value) - + + save_path = os.path.join( + Path(self.output_directory).absolute(), elem1.value + ) + save_path_exists = True if Path(save_path).exists() else False - elem1.label = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:" - elem1.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") - elem2.text = ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:" - elem2.tooltip ="" if not save_path_exists else (_validate_alert+"Output file exists") + elem1.label = ( + "" if not save_path_exists else (_validate_alert + " ") + ) + "Output Data:" + elem1.tooltip = ( + "" + if not save_path_exists + else (_validate_alert + "Output file exists") + ) + elem2.text = ( + "" if not save_path_exists else (_validate_alert + " ") + ) + "Output Data:" + elem2.tooltip = ( + "" + if not save_path_exists + else (_validate_alert + "Output file exists") + ) - self.saveLastPaths() + self.save_last_paths() # Copied from main_widget # ToDo: utilize common functions # Output data selector def browse_dir_path_model(self): - results = self._open_file_dialog( + results = self.open_file_dialog( self.directory, "files", filter="YAML Files (*.yml)" ) # returns list if len(results) == 0 or results == "": @@ -639,14 +739,13 @@ def browse_dir_path_model(self): self.directory = self.model_directory self.current_dir_path = self.model_directory - self.saveLastPaths() - self.openModelFiles(results) - + self.save_last_paths() + self.open_model_files(results) - def openModelFiles(self, results:List): + def open_model_files(self, results: List): pydantic_models = list() for result in results: - self.yaml_model_file = result + self.yaml_model_file = result with open(result, "r") as yaml_in: yaml_object = utils.yaml.safe_load( @@ -665,9 +764,9 @@ def openModelFiles(self, results:List): else: selected_modes.pop(k) - pruned_pydantic_class, ret_msg = self.buildModel(selected_modes) + pruned_pydantic_class, ret_msg = self.build_model(selected_modes) if pruned_pydantic_class is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_model, ret_msg = self.get_model_from_file( @@ -684,11 +783,11 @@ def openModelFiles(self, results:List): json_dict["birefringence"]["apply_inverse"][ "background_path" ] = "" - self.messageBox( + self.message_box( "background_path:\nPath was invalid and will be reset" ) else: - self.messageBox(ret_msg) + self.message_box(ret_msg) return else: # make sure "background_path" is valid @@ -706,10 +805,10 @@ def openModelFiles(self, results:List): ) else: bg_loc = Path(os.path.join(bg_loc, "background.zarr")) - if not bg_loc.exists() or not self.validateInputData( + if not bg_loc.exists() or not self.validate_input_data( str(bg_loc) ): - self.messageBox( + self.message_box( "background_path:\nPwas invalid and will be reset" ) json_dict["birefringence"]["apply_inverse"][ @@ -720,11 +819,11 @@ def openModelFiles(self, results:List): "background_path" ] = str(bg_loc.parent.absolute()) - pydantic_model = self._create_acq_contols2( + pydantic_model = self.create_acq_contols2( selected_modes, exclude_modes, pydantic_model, json_dict ) if pydantic_model is None: - self.messageBox("Error - pydantic model returned None") + self.message_box("Error - pydantic model returned None") return pydantic_models.append(pydantic_model) @@ -732,7 +831,7 @@ def openModelFiles(self, results:List): return pydantic_models # useful when using close widget and not napari close and we might need them again - def saveLastPaths(self): + def save_last_paths(self): HAS_INSTANCE["current_dir_path"] = self.current_dir_path HAS_INSTANCE["input_directory"] = self.input_directory HAS_INSTANCE["output_directory"] = self.output_directory @@ -743,13 +842,13 @@ def saveLastPaths(self): def clear_results_table(self): index = self.proc_table_QFormLayout.rowCount() if index < 1: - self.messageBox("There are no processing results to clear !") + self.message_box("There are no processing results to clear !") return - if self.confirmDialog(): + if self.confirm_dialog(): for i in range(self.proc_table_QFormLayout.rowCount()): self.proc_table_QFormLayout.removeRow(0) - def removeRow(self, row, expID): + def remove_row(self, row, expID): try: if row < self.proc_table_QFormLayout.rowCount(): widgetItem = self.proc_table_QFormLayout.itemAt(row) @@ -764,7 +863,7 @@ def removeRow(self, row, expID): print(exc.args) # marks fields on the Model that cause a validation error - def modelHighlighter(self, errs): + def model_highlighter(self, errs): try: for uid in errs.keys(): self.modelHighlighterVals[uid] = {} @@ -773,7 +872,7 @@ def modelHighlighter(self, errs): self.modelHighlighterVals[uid]["items"] = [] self.modelHighlighterVals[uid]["tooltip"] = [] if len(errs[uid]["errs"]) > 0: - self.modelHighlighterSetter( + self.model_highlighter_setter( errs[uid]["errs"], container, uid ) except Exception as exc: @@ -781,7 +880,7 @@ def modelHighlighter(self, errs): # more of a test feature - no need to show up # format all model errors into a display format for napari error message box - def formatStringForErrorDisplay(self, errs): + def format_string_for_error_display(self, errs): try: ret_str = "" for uid in errs.keys(): @@ -795,7 +894,7 @@ def formatStringForErrorDisplay(self, errs): return ret_str # recursively fix the container for highlighting - def modelHighlighterSetter( + def model_highlighter_setter( self, errs, container: Container, containerID, lev=0 ): try: @@ -817,7 +916,7 @@ def modelHighlighterSetter( ) and not (widget._magic_widget._inner_widget is None) ): - self.modelHighlighterSetter( + self.model_highlighter_setter( errs, widget._magic_widget._inner_widget, containerID, @@ -901,17 +1000,15 @@ def modelHighlighterSetter( ) except Exception as exc: print(exc.args) - # more of a test feature - no need to show up # recursively fix the container for highlighting - def modelResetHighlighterSetter(self): + def model_reset_highlighter_setter(self): try: for containerID in self.modelHighlighterVals.keys(): items = self.modelHighlighterVals[containerID]["items"] tooltip = self.modelHighlighterVals[containerID]["tooltip"] i = 0 for widItem in items: - # widItem.tooltip = None # let them tool tip remain widItem.native.setStyleSheet( "border:1px solid rgb(0, 0, 0); border-width: 0px;" ) @@ -920,14 +1017,12 @@ def modelResetHighlighterSetter(self): except Exception as exc: print(exc.args) - # more of a test feature - no need to show up except Exception as exc: print(exc.args) - # more of a test feature - no need to show up # passes msg to napari notifications - def messageBox(self, msg, type="exc"): + def message_box(self, msg, type="exc"): if len(msg) > 0: try: json_object = msg @@ -946,44 +1041,55 @@ def messageBox(self, msg, type="exc"): json_txt = str(msg) # show is a message box - if self.stand_alone: - self.messageBoxStandAlone(json_txt) + if self.stand_alone: + self.message_box_stand_alone(json_txt) else: if type == "exc": notifications.show_error(json_txt) else: notifications.show_info(json_txt) - - def messageBoxStandAlone(self, msg): - q = QMessageBox(QMessageBox.Warning, "Message", str(msg), parent=self.recon_tab_widget) + + def message_box_stand_alone(self, msg): + q = QMessageBox( + QMessageBox.Warning, + "Message", + str(msg), + parent=self.recon_tab_widget, + ) q.setStandardButtons(QMessageBox.StandardButton.Ok) q.setIcon(QMessageBox.Icon.Warning) q.exec_() - def cancelJob(self, btn:PushButton): - if self.confirmDialog(): + def cancel_job(self, btn: PushButton): + if self.confirm_dialog(): btn.enabled = False btn.text = btn.text + " (cancel called)" - def add_widget(self, parentLayout:QVBoxLayout, expID, jID, tableEntryID="", pos=""): - + def add_widget( + self, parentLayout: QVBoxLayout, expID, jID, table_entry_ID="", pos="" + ): + jID = str(jID) - _cancelJobBtntext = "Cancel Job {jID} ({posName})".format(jID=jID, posName=pos) + _cancelJobBtntext = "Cancel Job {jID} ({posName})".format( + jID=jID, posName=pos + ) _cancelJobButton = widgets.PushButton( name="JobID", label=_cancelJobBtntext, enabled=True, value=False ) _cancelJobButton.clicked.connect( - lambda: self.cancelJob(_cancelJobButton) + lambda: self.cancel_job(_cancelJobButton) ) _txtForInfoBox = "Updating {id}-{pos}: Please wait... \nJobID assigned: {jID} ".format( - id=tableEntryID, jID=jID, pos=pos - ) + id=table_entry_ID, jID=jID, pos=pos + ) _scrollAreaCollapsibleBoxDisplayWidget = ScrollableLabel( text=_txtForInfoBox ) _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() - _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + _scrollAreaCollapsibleBoxWidgetLayout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) _scrollAreaCollapsibleBoxWidgetLayout.addWidget( _cancelJobButton.native @@ -1005,7 +1111,7 @@ def add_widget(self, parentLayout:QVBoxLayout, expID, jID, tableEntryID="", pos= _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) _collapsibleBoxWidget = CollapsibleBox( - tableEntryID + " - " + pos + table_entry_ID + " - " + pos ) # tableEntryID, tableEntryShortDesc - should update with processing status _collapsibleBoxWidget.setSizePolicy( QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed @@ -1015,21 +1121,23 @@ def add_widget(self, parentLayout:QVBoxLayout, expID, jID, tableEntryID="", pos= parentLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) parentLayout.addWidget(_collapsibleBoxWidget) - MULTI_JOBS_REFS[expID+jID] = {} - MULTI_JOBS_REFS[expID+jID]["cancelBtn"] = _cancelJobButton - MULTI_JOBS_REFS[expID+jID]["infobox"] = _scrollAreaCollapsibleBoxDisplayWidget - NEW_WIDGETS_QUEUE.remove(expID+jID) + MULTI_JOBS_REFS[expID + jID] = {} + MULTI_JOBS_REFS[expID + jID]["cancelBtn"] = _cancelJobButton + MULTI_JOBS_REFS[expID + jID][ + "infobox" + ] = _scrollAreaCollapsibleBoxDisplayWidget + NEW_WIDGETS_QUEUE.remove(expID + jID) - def addTableEntryJob(self, proc_params): + def add_table_entry_job(self, proc_params): tableEntryID = proc_params["tableEntryID"] - parentLayout:QVBoxLayout = proc_params["parent_layout"] - + parentLayout: QVBoxLayout = proc_params["parent_layout"] + _cancelJobButton = widgets.PushButton( name="JobID", label="Cancel Job", value=False, enabled=False ) _cancelJobButton.clicked.connect( - lambda: self.cancelJob(_cancelJobButton) + lambda: self.cancel_job(_cancelJobButton) ) _txtForInfoBox = "Updating {id}: Please wait...".format( id=tableEntryID @@ -1038,45 +1146,49 @@ def addTableEntryJob(self, proc_params): text=_txtForInfoBox ) _scrollAreaCollapsibleBoxDisplayWidget.setFixedHeight(300) - + proc_params["table_entry_infoBox"] = ( _scrollAreaCollapsibleBoxDisplayWidget ) - proc_params["cancelJobButton"] = ( - _cancelJobButton - ) - parentLayout.addWidget( - _cancelJobButton.native - ) - parentLayout.addWidget( - _scrollAreaCollapsibleBoxDisplayWidget - ) - + proc_params["cancelJobButton"] = _cancelJobButton + parentLayout.addWidget(_cancelJobButton.native) + parentLayout.addWidget(_scrollAreaCollapsibleBoxDisplayWidget) + return proc_params - def addRemoveCheckOTFTableEntry(self, OTF_dir_path, bool_msg, doCheck=False): - if doCheck: + def add_remove_check_OTF_table_entry( + self, OTF_dir_path, bool_msg, do_check=False + ): + if do_check: try: for row in range(self.proc_OTF_table_QFormLayout.rowCount()): widgetItem = self.proc_OTF_table_QFormLayout.itemAt(row) if widgetItem is not None: - name_widget:QWidget = widgetItem.widget() + name_widget: QWidget = widgetItem.widget() name_string = str(name_widget.objectName()) if OTF_dir_path in name_string: for item in name_widget.findChildren(QPushButton): - _poll_Stop_PushButton:QPushButton = item + _poll_Stop_PushButton: QPushButton = item return _poll_Stop_PushButton.isChecked() - return False + return False except Exception as exc: print(exc.args) return False else: if bool_msg: - _poll_otf_label = ScrollableLabel(text=OTF_dir_path + " " + _green_dot) + _poll_otf_label = ScrollableLabel( + text=OTF_dir_path + " " + _green_dot + ) _poll_Stop_PushButton = QPushButton("Stop") - _poll_Stop_PushButton.setCheckable(True) # Make the button checkable - _poll_Stop_PushButton.clicked.connect(lambda:self.stopOTF_PushButtonCall(_poll_otf_label, OTF_dir_path + " " + _red_dot)) - + _poll_Stop_PushButton.setCheckable( + True + ) # Make the button checkable + _poll_Stop_PushButton.clicked.connect( + lambda: self.stop_OTF_push_button_call( + _poll_otf_label, OTF_dir_path + " " + _red_dot + ) + ) + _poll_data_widget = QWidget() _poll_data_widget.setObjectName(OTF_dir_path) _poll_data_widget_layout = QHBoxLayout() @@ -1087,17 +1199,21 @@ def addRemoveCheckOTFTableEntry(self, OTF_dir_path, bool_msg, doCheck=False): self.proc_OTF_table_QFormLayout.insertRow(0, _poll_data_widget) else: try: - for row in range(self.proc_OTF_table_QFormLayout.rowCount()): - widgetItem = self.proc_OTF_table_QFormLayout.itemAt(row) + for row in range( + self.proc_OTF_table_QFormLayout.rowCount() + ): + widgetItem = self.proc_OTF_table_QFormLayout.itemAt( + row + ) if widgetItem is not None: - name_widget:QWidget = widgetItem.widget() + name_widget: QWidget = widgetItem.widget() name_string = str(name_widget.objectName()) if OTF_dir_path in name_string: self.proc_OTF_table_QFormLayout.removeRow(row) except Exception as exc: print(exc.args) - def stopOTF_PushButtonCall(self, label, txt): + def stop_OTF_push_button_call(self, label, txt): _poll_otf_label: QLabel = label _poll_otf_label.setText(txt) self.setDisabled(True) @@ -1106,11 +1222,11 @@ def stopOTF_PushButtonCall(self, label, txt): # row item will be purged from table as processing finishes # there could be 3 tabs for this processing table status # Running, Finished, Errored - def addTableEntry( - self, tableEntryID, tableEntryShortDesc, proc_params - ): + def addTableEntry(self, table_entry_ID, table_entry_short_desc, proc_params): _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() - _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + _scrollAreaCollapsibleBoxWidgetLayout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) _scrollAreaCollapsibleBoxWidget = QWidget() _scrollAreaCollapsibleBoxWidget.setLayout( @@ -1131,24 +1247,28 @@ def addTableEntry( _collapsibleBoxWidgetLayout = QVBoxLayout() _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) - _collapsibleBoxWidget = CollapsibleBox(tableEntryID) + _collapsibleBoxWidget = CollapsibleBox(table_entry_ID) _collapsibleBoxWidget.setSizePolicy( QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed ) _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) _expandingTabEntryWidgetLayout = QVBoxLayout() - _expandingTabEntryWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + _expandingTabEntryWidgetLayout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) _expandingTabEntryWidgetLayout.addWidget(_collapsibleBoxWidget) _expandingTabEntryWidget = QWidget() - _expandingTabEntryWidget.toolTip = tableEntryShortDesc + _expandingTabEntryWidget.toolTip = table_entry_short_desc _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) + _expandingTabEntryWidget.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) - proc_params["tableEntryID"] = tableEntryID + proc_params["tableEntryID"] = table_entry_ID proc_params["parent_layout"] = _scrollAreaCollapsibleBoxWidgetLayout - proc_params = self.addTableEntryJob(proc_params) + proc_params = self.add_table_entry_job(proc_params) # instead of adding, insert at 0 to keep latest entry on top # self.proc_table_QFormLayout.addRow(_expandingTabEntryWidget) @@ -1156,48 +1276,50 @@ def addTableEntry( proc_params["table_layout"] = self.proc_table_QFormLayout proc_params["table_entry"] = _expandingTabEntryWidget - - self.worker.runInPool(proc_params) + + self.worker.run_in_pool(proc_params) # result = self.worker.getResult(proc_params["exp_id"]) # print(result) # Builds the model as required - def buildModel(self, selected_modes): + def build_model(self, selected_modes): try: - b = None - p = None - f = None + birefringence = None + phase = None + fluorescence = None chNames = ["State0"] exclude_modes = ["birefringence", "phase", "fluorescence"] if "birefringence" in selected_modes and "phase" in selected_modes: - b = settings.BirefringenceSettings() - p = settings.PhaseSettings() + birefringence = settings.BirefringenceSettings() + phase = settings.PhaseSettings() chNames = ["State0", "State1", "State2", "State3"] exclude_modes = ["fluorescence"] elif "birefringence" in selected_modes: - b = settings.BirefringenceSettings() + birefringence = settings.BirefringenceSettings() chNames = ["State0", "State1", "State2", "State3"] exclude_modes = ["fluorescence", "phase"] elif "phase" in selected_modes: - p = settings.PhaseSettings() + phase = settings.PhaseSettings() + chNames = ["BF"] exclude_modes = ["birefringence", "fluorescence"] elif "fluorescence" in selected_modes: - f = settings.FluorescenceSettings() + fluorescence = settings.FluorescenceSettings() + chNames = ["FL"] exclude_modes = ["birefringence", "phase"] model = None try: model = settings.ReconstructionSettings( input_channel_names=chNames, - birefringence=b, - phase=p, - fluorescence=f, + birefringence=birefringence, + phase=phase, + fluorescence=fluorescence, ) except ValidationError as exc: # use v1 and v2 differ for ValidationError - newer one is not caught properly return None, exc.errors() - model = self._fix_model( + model = self.fix_model( model, exclude_modes, "input_channel_names", chNames ) return model, "+".join(selected_modes) + ": MSG_SUCCESS" @@ -1207,7 +1329,7 @@ def buildModel(self, selected_modes): # ToDo: Temporary fix to over ride the 'input_channel_names' default value # Needs revisitation - def _fix_model(self, model, exclude_modes, attr_key, attr_val): + def fix_model(self, model, exclude_modes, attr_key, attr_val): try: for mode in exclude_modes: model = settings.ReconstructionSettings.copy( @@ -1227,7 +1349,7 @@ def _fix_model(self, model, exclude_modes, attr_key, attr_val): return model # Creates UI controls from model based on selections - def _build_acq_contols(self): + def build_acq_contols(self): # Make a copy of selections and unsed for deletion selected_modes = [] @@ -1240,14 +1362,16 @@ def _build_acq_contols(self): else: selected_modes.append(mode) - self._create_acq_contols2(selected_modes, exclude_modes) + self.create_acq_contols2(selected_modes, exclude_modes) - def _create_acq_contols2( - self, selected_modes, exclude_modes, myLoadedModel=None, json_dict=None + def create_acq_contols2( + self, selected_modes, exclude_modes, my_loaded_model=None, json_dict=None ): # duplicate settings from the prev model on new model creation if json_dict is None and len(self.pydantic_classes) > 0: - ret = self.build_model_and_run(validate_return_prev_model_json_txt=True) + ret = self.build_model_and_run( + validate_return_prev_model_json_txt=True + ) if ret is None: return key, json_txt = ret @@ -1258,12 +1382,12 @@ def _create_acq_contols2( json_dict = self.prev_model_settings[key] # initialize the top container and specify what pydantic class to map from - if myLoadedModel is not None: - pydantic_class = myLoadedModel + if my_loaded_model is not None: + pydantic_class = my_loaded_model else: - pydantic_class, ret_msg = self.buildModel(selected_modes) + pydantic_class, ret_msg = self.build_model(selected_modes) if pydantic_class is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # Final constant UI val and identifier @@ -1291,7 +1415,7 @@ def _create_acq_contols2( exclude_modes, ) if pydantic_kwargs is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # For list element, this needs to be cleaned and parsed back as an array @@ -1299,7 +1423,7 @@ def _create_acq_contols2( "input_channel_names", pydantic_kwargs["input_channel_names"] ) if input_channel_names is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["input_channel_names"] = input_channel_names @@ -1307,7 +1431,7 @@ def _create_acq_contols2( "time_indices", pydantic_kwargs["time_indices"] ) if time_indices is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["time_indices"] = time_indices @@ -1319,7 +1443,7 @@ def _create_acq_contols2( ], ) if background_path is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["birefringence"]["apply_inverse"][ "background_path" @@ -1330,14 +1454,14 @@ def _create_acq_contols2( pydantic_class, pydantic_kwargs ) if pydantic_model is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # generate a json from the instantiated model, update the json_display # most of this will end up in a table as processing proceeds json_txt, ret_msg = self.validate_and_return_json(pydantic_model) if json_txt is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # PushButton to delete a UI container @@ -1347,7 +1471,9 @@ def _create_acq_contols2( _del_button = widgets.PushButton(name="Delete Model") c_mode = "-and-".join(selected_modes) - c_mode_short = "".join(item[:3].capitalize() for item in selected_modes) + c_mode_short = "".join( + item[:3].capitalize() for item in selected_modes + ) if c_mode in CONTAINERS_INFO.keys(): CONTAINERS_INFO[c_mode] += 1 else: @@ -1359,13 +1485,31 @@ def _create_acq_contols2( # These could be multiple based on user selection for each model # Inherits from Input by default at creation time name_without_ext = os.path.splitext(Path(self.input_directory).name)[0] - save_path = os.path.join(Path(self.output_directory).absolute(), (name_without_ext + ("_"+c_mode_short+"_"+num_str) + ".zarr")) + save_path = os.path.join( + Path(self.output_directory).absolute(), + ( + name_without_ext + + ("_" + c_mode_short + "_" + num_str) + + ".zarr" + ), + ) save_path_exists = True if Path(save_path).exists() else False _output_data_loc = widgets.LineEdit( - value=Path(save_path).name, tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") + value=Path(save_path).name, + tooltip=( + "" + if not save_path_exists + else (_validate_alert + " Output file exists") + ), ) _output_data_btn = widgets.PushButton( - text= ("" if not save_path_exists else (_validate_alert+" ")) + "Output Data:", tooltip="" if not save_path_exists else (_validate_alert+" Output file exists") + text=("" if not save_path_exists else (_validate_alert + " ")) + + "Output Data:", + tooltip=( + "" + if not save_path_exists + else (_validate_alert + " Output file exists") + ), ) # Passing location label to output location selector @@ -1373,20 +1517,24 @@ def _create_acq_contols2( lambda: self.browse_model_dir_path_output(_output_data_loc) ) _output_data_loc.changed.connect( - lambda: self.readAndSetOutputPathOnValidation(_output_data_loc, _output_data_btn, save_path) + lambda: self.read_and_set_output_path_on_validation( + _output_data_loc, _output_data_btn, save_path + ) ) - _show_CheckBox = widgets.CheckBox(name="Show after Reconstruction", value=True) + _show_CheckBox = widgets.CheckBox( + name="Show after Reconstruction", value=True + ) _show_CheckBox.max_width = 200 _rx_Label = widgets.Label(value="rx") _rx_LineEdit = widgets.LineEdit(name="rx", value=1) _rx_LineEdit.max_width = 50 - _validate_button = widgets.PushButton(name="Validate") + _validate_button = widgets.PushButton(name="Validate") # Passing all UI components that would be deleted _expandingTabEntryWidget = QWidget() _del_button.clicked.connect( - lambda: self._delete_model( + lambda: self.delete_model( _expandingTabEntryWidget, recon_pydantic_container.native, _output_data_loc.native, @@ -1402,13 +1550,15 @@ def _create_acq_contols2( _hBox_widget = QWidget() _hBox_layout = QHBoxLayout() _hBox_layout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) - _hBox_widget.setLayout(_hBox_layout) + _hBox_widget.setLayout(_hBox_layout) _hBox_layout.addWidget(_output_data_btn.native) _hBox_layout.addWidget(_output_data_loc.native) # Add this container to the main scrollable widget _scrollAreaCollapsibleBoxWidgetLayout = QVBoxLayout() - _scrollAreaCollapsibleBoxWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + _scrollAreaCollapsibleBoxWidgetLayout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) _scrollAreaCollapsibleBoxWidget = MyWidget() _scrollAreaCollapsibleBoxWidget.setLayout( @@ -1418,53 +1568,69 @@ def _create_acq_contols2( _scrollAreaCollapsibleBox = QScrollArea() _scrollAreaCollapsibleBox.setWidgetResizable(True) _scrollAreaCollapsibleBox.setWidget(_scrollAreaCollapsibleBoxWidget) - + _collapsibleBoxWidgetLayout = QVBoxLayout() _collapsibleBoxWidgetLayout.setAlignment(Qt.AlignmentFlag.AlignTop) - - scrollbar = _scrollAreaCollapsibleBox.horizontalScrollBar() - _scrollAreaCollapsibleBoxWidget.resized.connect(lambda:self.check_scrollbar_visibility(scrollbar)) - _scrollAreaCollapsibleBoxWidgetLayout.addWidget(scrollbar, alignment=Qt.AlignmentFlag.AlignTop) # Place at the top + scrollbar = _scrollAreaCollapsibleBox.horizontalScrollBar() + _scrollAreaCollapsibleBoxWidget.resized.connect( + lambda: self.check_scrollbar_visibility(scrollbar) + ) + + _scrollAreaCollapsibleBoxWidgetLayout.addWidget( + scrollbar, alignment=Qt.AlignmentFlag.AlignTop + ) # Place at the top _collapsibleBoxWidgetLayout.addWidget(_scrollAreaCollapsibleBox) _collapsibleBoxWidget = CollapsibleBox( c_mode_str ) # tableEntryID, tableEntryShortDesc - should update with processing status - - _validate_button.clicked.connect(lambda:self._validate_model(_str, _collapsibleBoxWidget)) + + _validate_button.clicked.connect( + lambda: self.validate_model(_str, _collapsibleBoxWidget) + ) _hBox_widget2 = QWidget() _hBox_layout2 = QHBoxLayout() _hBox_layout2.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) _hBox_widget2.setLayout(_hBox_layout2) - _hBox_layout2.addWidget(_show_CheckBox.native) + _hBox_layout2.addWidget(_show_CheckBox.native) _hBox_layout2.addWidget(_validate_button.native) _hBox_layout2.addWidget(_del_button.native) _hBox_layout2.addWidget(_rx_Label.native) _hBox_layout2.addWidget(_rx_LineEdit.native) _expandingTabEntryWidgetLayout = QVBoxLayout() - _expandingTabEntryWidgetLayout.setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + _expandingTabEntryWidgetLayout.setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) _expandingTabEntryWidgetLayout.addWidget(_collapsibleBoxWidget) _expandingTabEntryWidget.toolTip = c_mode_str _expandingTabEntryWidget.setLayout(_expandingTabEntryWidgetLayout) - _expandingTabEntryWidget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) - _expandingTabEntryWidget.layout().setAlignment(QtCore.Qt.AlignmentFlag.AlignTop) + _expandingTabEntryWidget.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) + _expandingTabEntryWidget.layout().setAlignment( + QtCore.Qt.AlignmentFlag.AlignTop + ) - _scrollAreaCollapsibleBoxWidgetLayout.addWidget(recon_pydantic_container.native) + _scrollAreaCollapsibleBoxWidgetLayout.addWidget( + recon_pydantic_container.native + ) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget) _scrollAreaCollapsibleBoxWidgetLayout.addWidget(_hBox_widget2) - _scrollAreaCollapsibleBox.setMinimumHeight(_scrollAreaCollapsibleBoxWidgetLayout.sizeHint().height()+20) - _collapsibleBoxWidget.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) + _scrollAreaCollapsibleBox.setMinimumHeight( + _scrollAreaCollapsibleBoxWidgetLayout.sizeHint().height() + 20 + ) + _collapsibleBoxWidget.setSizePolicy( + QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed + ) _collapsibleBoxWidget.setContentLayout(_collapsibleBoxWidgetLayout) - self.models_container_widget_layout.addWidget( - _expandingTabEntryWidget - ) + self.models_container_widget_layout.addWidget(_expandingTabEntryWidget) # Store a copy of the pydantic container along with all its associated components and properties # We dont needs a copy of the class but storing for now @@ -1477,8 +1643,13 @@ def _create_acq_contols2( "collapsibleBoxWidget": _collapsibleBoxWidget, "class": pydantic_class, "input": self.data_input_LineEdit, - "output": os.path.join(Path(self.output_directory).absolute(), _output_data_loc.value), - "output_parent_dir": str(Path(self.output_directory).absolute()), + "output": os.path.join( + Path(self.output_directory).absolute(), + _output_data_loc.value, + ), + "output_parent_dir": str( + Path(self.output_directory).absolute() + ), "output_LineEdit": _output_data_loc, "output_Button": _output_data_btn, "container": recon_pydantic_container, @@ -1486,7 +1657,7 @@ def _create_acq_contols2( "exclude_modes": exclude_modes.copy(), "poll_data": self.pollData, "show": _show_CheckBox, - "rx":_rx_LineEdit, + "rx": _rx_LineEdit, } ) self.index += 1 @@ -1499,14 +1670,14 @@ def _create_acq_contols2( self.reconstruction_run_PushButton.text = "RUN Model" return pydantic_model - + def check_scrollbar_visibility(self, scrollbar): h_scrollbar = scrollbar # Hide scrollbar if not needed h_scrollbar.setVisible(h_scrollbar.maximum() > h_scrollbar.minimum()) - def _validate_model(self, _str, _collapsibleBoxWidget): + def validate_model(self, _str, _collapsibleBoxWidget): i = 0 model_entry_item = None for item in self.pydantic_classes: @@ -1522,7 +1693,7 @@ def _validate_model(self, _str, _collapsibleBoxWidget): # build up the arguments for the pydantic model given the current container if cls is None: - self.messageBox("No model defined !") + self.message_box("No model defined !") return pydantic_kwargs = {} @@ -1530,7 +1701,7 @@ def _validate_model(self, _str, _collapsibleBoxWidget): cls_container, cls, pydantic_kwargs, exclude_modes ) if pydantic_kwargs is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) _collapsibleBoxWidget.setNewName( f"{c_mode_str} {_validate_alert}" ) @@ -1540,7 +1711,7 @@ def _validate_model(self, _str, _collapsibleBoxWidget): "input_channel_names", pydantic_kwargs["input_channel_names"] ) if input_channel_names is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) _collapsibleBoxWidget.setNewName( f"{c_mode_str} {_validate_alert}" ) @@ -1551,7 +1722,7 @@ def _validate_model(self, _str, _collapsibleBoxWidget): "time_indices", pydantic_kwargs["time_indices"] ) if time_indices is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) _collapsibleBoxWidget.setNewName( f"{c_mode_str} {_validate_alert}" ) @@ -1562,7 +1733,7 @@ def _validate_model(self, _str, _collapsibleBoxWidget): "time_indices", pydantic_kwargs["time_indices"] ) if time_indices is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) _collapsibleBoxWidget.setNewName( f"{c_mode_str} {_validate_alert}" ) @@ -1577,7 +1748,7 @@ def _validate_model(self, _str, _collapsibleBoxWidget): ], ) if background_path is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) _collapsibleBoxWidget.setNewName( f"{c_mode_str} {_validate_alert}" ) @@ -1591,7 +1762,7 @@ def _validate_model(self, _str, _collapsibleBoxWidget): cls, pydantic_kwargs ) if pydantic_model is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) _collapsibleBoxWidget.setNewName( f"{c_mode_str} {_validate_alert}" ) @@ -1605,11 +1776,10 @@ def _validate_model(self, _str, _collapsibleBoxWidget): f"{c_mode_str} {_validate_alert}" ) - # UI components deletion - maybe just needs the parent container instead of individual components - def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, wid6, _str): + def delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, wid6, _str): - if not self.confirmDialog(): + if not self.confirm_dialog(): return False # if wid5 is not None: @@ -1641,9 +1811,9 @@ def _delete_model(self, wid0, wid1, wid2, wid3, wid4, wid5, wid6, _str): self.reconstruction_run_PushButton.text = "RUN Model" # Clear all the generated pydantic models and clears the pydantic model list - def _clear_all_models(self, silent=False): - - if silent or self.confirmDialog(): + def clear_all_models(self, silent=False): + + if silent or self.confirm_dialog(): index = self.models_container_widget_layout.count() - 1 while index >= 0: myWidget = self.models_container_widget_layout.itemAt( @@ -1667,10 +1837,10 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): # second pass for creating yaml and processing if len(self.pydantic_classes) == 0: - self.messageBox("Please create a processing model first !") + self.message_box("Please create a processing model first !") return - self.modelResetHighlighterSetter() # reset the container elements that might be highlighted for errors + self.model_reset_highlighter_setter() # reset the container elements that might be highlighted for errors _collectAllErrors = {} _collectAllErrorsBool = True for item in self.pydantic_classes: @@ -1689,7 +1859,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): # build up the arguments for the pydantic model given the current container if cls is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # get the kwargs from the container/class @@ -1698,7 +1868,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): cls_container, cls, pydantic_kwargs, exclude_modes ) if pydantic_kwargs is None and not _collectAllErrorsBool: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # For list element, this needs to be cleaned and parsed back as an array @@ -1706,7 +1876,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): "input_channel_names", pydantic_kwargs["input_channel_names"] ) if input_channel_names is None and not _collectAllErrorsBool: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["input_channel_names"] = input_channel_names @@ -1714,7 +1884,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): "time_indices", pydantic_kwargs["time_indices"] ) if time_indices is None and not _collectAllErrorsBool: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["time_indices"] = time_indices @@ -1726,7 +1896,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): ], ) if background_path is None and not _collectAllErrorsBool: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["birefringence"]["apply_inverse"][ "background_path" @@ -1746,24 +1916,24 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): ) _collectAllErrors[uuid_str]["errs"] = ret_msg if pydantic_model is None and not _collectAllErrorsBool: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # generate a json from the instantiated model, update the json_display # most of this will end up in a table as processing proceeds json_txt, ret_msg = self.validate_and_return_json(pydantic_model) if json_txt is None and not _collectAllErrorsBool: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # check if we collected any validation errors before continuing for uu_key in _collectAllErrors.keys(): if len(_collectAllErrors[uu_key]["errs"]) > 0: - self.modelHighlighter(_collectAllErrors) - fmt_str = self.formatStringForErrorDisplay(_collectAllErrors) - self.messageBox(fmt_str) + self.model_highlighter(_collectAllErrors) + fmt_str = self.format_string_for_error_display(_collectAllErrors) + self.message_box(fmt_str) return - + if validate_return_prev_model_json_txt: return "-".join(selected_modes), json_txt @@ -1776,14 +1946,17 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): if self.pollData: data = open_ome_zarr(self.input_directory, mode="r") if "CurrentDimensions" in data.zattrs.keys(): - my_dict_time_indices = data.zattrs["CurrentDimensions"]["time"] + my_dict_time_indices = data.zattrs["CurrentDimensions"]["time"] # get the prev time_index, since this is current acq - if my_dict_time_indices-1 > 1: + if my_dict_time_indices - 1 > 1: time_indices = list(range(0, my_dict_time_indices)) else: time_indices = 0 - pollDataThread = threading.Thread(target=self.addPollLoop, args=(self.input_directory, my_dict_time_indices-1),) + pollDataThread = threading.Thread( + target=self.add_poll_loop, + args=(self.input_directory, my_dict_time_indices - 1), + ) pollDataThread.start() i = 0 @@ -1797,7 +1970,9 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): output_LineEdit = item["output_LineEdit"] output_parent_dir = item["output_parent_dir"] - full_out_path = os.path.join(output_parent_dir, output_LineEdit.value) + full_out_path = os.path.join( + output_parent_dir, output_LineEdit.value + ) # gather input/out locations input_dir = f"{item['input'].value}" @@ -1805,7 +1980,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): # build up the arguments for the pydantic model given the current container if cls is None: - self.messageBox("No model defined !") + self.message_box("No model defined !") return pydantic_kwargs = {} @@ -1813,14 +1988,14 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): cls_container, cls, pydantic_kwargs, exclude_modes ) if pydantic_kwargs is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return input_channel_names, ret_msg = self.clean_string_for_list( "input_channel_names", pydantic_kwargs["input_channel_names"] ) if input_channel_names is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["input_channel_names"] = input_channel_names @@ -1829,7 +2004,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): "time_indices", pydantic_kwargs["time_indices"] ) if time_indices is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["time_indices"] = time_indices @@ -1837,7 +2012,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): "time_indices", pydantic_kwargs["time_indices"] ) if time_indices is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["time_indices"] = time_indices @@ -1849,7 +2024,7 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): ], ) if background_path is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return pydantic_kwargs["birefringence"]["apply_inverse"][ "background_path" @@ -1860,21 +2035,23 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): cls, pydantic_kwargs ) if pydantic_model is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # generate a json from the instantiated model, update the json_display # most of this will end up in a table as processing proceeds json_txt, ret_msg = self.validate_and_return_json(pydantic_model) if json_txt is None: - self.messageBox(ret_msg) + self.message_box(ret_msg) return # save the yaml files # path is next to saved data location save_config_path = str(Path(output_dir).parent.absolute()) yml_file_name = "-and-".join(selected_modes) - yml_file = yml_file_name + "-" + unique_id + "-{:02d}".format(i) + ".yml" + yml_file = ( + yml_file_name + "-" + unique_id + "-{:02d}".format(i) + ".yml" + ) config_path = os.path.join(save_config_path, yml_file) utils.model_to_yaml(pydantic_model, config_path) @@ -1901,31 +2078,41 @@ def build_model_and_run(self, validate_return_prev_model_json_txt=False): proc_params["config_path"] = str(Path(config_path).absolute()) proc_params["input_path"] = str(Path(input_dir).absolute()) proc_params["output_path"] = str(Path(output_dir).absolute()) - proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) + proc_params["output_path_parent"] = str( + Path(output_dir).parent.absolute() + ) proc_params["show"] = item["show"].value proc_params["rx"] = item["rx"].value - self.addTableEntry( - tableID, tableDescToolTip, proc_params - ) + self.addTableEntry(tableID, tableDescToolTip, proc_params) - def addPollLoop(self, input_data_path, last_time_index): + def add_poll_loop(self, input_data_path, last_time_index): _pydantic_classes = self.pydantic_classes.copy() - required_order = ['time', 'position', 'z', 'channel'] + required_order = ["time", "position", "z", "channel"] _pollData = True - tableEntryWorker = AddOTFTableEntryWorkerThread(input_data_path, True, False) - tableEntryWorker.add_tableOTFentry_signal.connect(self.addRemoveCheckOTFTableEntry) + tableEntryWorker = AddOTFTableEntryWorkerThread( + input_data_path, True, False + ) + tableEntryWorker.add_tableOTFentry_signal.connect( + self.add_remove_check_OTF_table_entry + ) tableEntryWorker.start() _breakFlag = False while True: time.sleep(10) zattrs_data = None try: - _stopCalled = self.addRemoveCheckOTFTableEntry(input_data_path, True, doCheck=True) - if _stopCalled: - tableEntryWorker2 = AddOTFTableEntryWorkerThread(input_data_path, False, False) - tableEntryWorker2.add_tableOTFentry_signal.connect(self.addRemoveCheckOTFTableEntry) + _stopCalled = self.add_remove_check_OTF_table_entry( + input_data_path, True, do_check=True + ) + if _stopCalled: + tableEntryWorker2 = AddOTFTableEntryWorkerThread( + input_data_path, False, False + ) + tableEntryWorker2.add_tableOTFentry_signal.connect( + self.add_remove_check_OTF_table_entry + ) tableEntryWorker2.start() # let child threads finish their work before exiting the parent thread @@ -1937,30 +2124,48 @@ def addPollLoop(self, input_data_path, last_time_index): data = open_ome_zarr(input_data_path, mode="r") zattrs_data = data.zattrs except PermissionError: - pass # On-The-Fly dataset will throw Permission Denied when being written - # Maybe we can read the zaatrs directly in that case - # If this write/read is a constant issue then the zattrs 'CurrentDimensions' key - # should be updated less frequently, instead of current design of updating with - # each image + pass # On-The-Fly dataset will throw Permission Denied when being written + # Maybe we can read the zaatrs directly in that case + # If this write/read is a constant issue then the zattrs 'CurrentDimensions' key + # should be updated less frequently, instead of current design of updating with + # each image if zattrs_data is None: - zattrs_data = self.loadZattrsDirectlyAsDict(input_data_path) + zattrs_data = self.load_zattrs_directly_as_dict( + input_data_path + ) if zattrs_data is not None: if "CurrentDimensions" in zattrs_data.keys(): my_dict1 = zattrs_data["CurrentDimensions"] - sorted_dict_acq = {k: my_dict1[k] for k in sorted(my_dict1, key=lambda x: required_order.index(x))} - my_dict_time_indices_curr = zattrs_data["CurrentDimensions"]["time"] + sorted_dict_acq = { + k: my_dict1[k] + for k in sorted( + my_dict1, key=lambda x: required_order.index(x) + ) + } + my_dict_time_indices_curr = zattrs_data[ + "CurrentDimensions" + ]["time"] # print(sorted_dict_acq) - + if "FinalDimensions" in zattrs_data.keys(): my_dict2 = zattrs_data["FinalDimensions"] - sorted_dict_final = {k: my_dict2[k] for k in sorted(my_dict2, key=lambda x: required_order.index(x))} + sorted_dict_final = { + k: my_dict2[k] + for k in sorted( + my_dict2, key=lambda x: required_order.index(x) + ) + } # print(sorted_dict_final) # use the prev time_index, since this is current acq and we need for other dims to finish acq for this t # or when all dims match - signifying acq finished - if my_dict_time_indices_curr-2 > last_time_index or json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): + if ( + my_dict_time_indices_curr - 2 > last_time_index + or json.dumps(sorted_dict_acq) + == json.dumps(sorted_dict_final) + ): now = datetime.datetime.now() ms = now.strftime("%f")[:3] @@ -1977,52 +2182,90 @@ def addPollLoop(self, input_data_path, last_time_index): output_LineEdit = item["output_LineEdit"] output_parent_dir = item["output_parent_dir"] - full_out_path = os.path.join(output_parent_dir, output_LineEdit.value) + full_out_path = os.path.join( + output_parent_dir, output_LineEdit.value + ) # gather input/out locations input_dir = f"{item['input'].value}" output_dir = full_out_path pydantic_kwargs = {} - pydantic_kwargs, ret_msg = self.get_and_validate_pydantic_args( - cls_container, cls, pydantic_kwargs, exclude_modes + pydantic_kwargs, ret_msg = ( + self.get_and_validate_pydantic_args( + cls_container, + cls, + pydantic_kwargs, + exclude_modes, + ) ) - input_channel_names, ret_msg = self.clean_string_for_list( - "input_channel_names", pydantic_kwargs["input_channel_names"] + input_channel_names, ret_msg = ( + self.clean_string_for_list( + "input_channel_names", + pydantic_kwargs["input_channel_names"], + ) + ) + pydantic_kwargs["input_channel_names"] = ( + input_channel_names ) - pydantic_kwargs["input_channel_names"] = input_channel_names - if _pollData: - if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final): - time_indices = list(range(last_time_index, my_dict_time_indices_curr)) + if _pollData: + if json.dumps(sorted_dict_acq) == json.dumps( + sorted_dict_final + ): + time_indices = list( + range( + last_time_index, + my_dict_time_indices_curr, + ) + ) _breakFlag = True else: - time_indices = list(range(last_time_index, my_dict_time_indices_curr-2)) + time_indices = list( + range( + last_time_index, + my_dict_time_indices_curr - 2, + ) + ) pydantic_kwargs["time_indices"] = time_indices if "birefringence" in pydantic_kwargs.keys(): - background_path, ret_msg = self.clean_path_string_when_empty( - "background_path", - pydantic_kwargs["birefringence"]["apply_inverse"][ - "background_path" - ], + background_path, ret_msg = ( + self.clean_path_string_when_empty( + "background_path", + pydantic_kwargs["birefringence"][ + "apply_inverse" + ]["background_path"], + ) ) - - pydantic_kwargs["birefringence"]["apply_inverse"][ - "background_path" - ] = background_path + + pydantic_kwargs["birefringence"][ + "apply_inverse" + ]["background_path"] = background_path # validate and return errors if None - pydantic_model, ret_msg = self.validate_pydantic_model( - cls, pydantic_kwargs + pydantic_model, ret_msg = ( + self.validate_pydantic_model( + cls, pydantic_kwargs + ) ) # save the yaml files # path is next to saved data location - save_config_path = str(Path(output_dir).parent.absolute()) + save_config_path = str( + Path(output_dir).parent.absolute() + ) yml_file_name = "-and-".join(selected_modes) - yml_file = yml_file_name + "-" + unique_id + "-{:02d}".format(i) + ".yml" - config_path = os.path.join(save_config_path, yml_file) + yml_file = ( + yml_file_name + + "-" + + unique_id + + "-{:02d}".format(i) + + ".yml" + ) + config_path = os.path.join( + save_config_path, yml_file + ) utils.model_to_yaml(pydantic_model, config_path) expID = "{tID}-{idx}".format(tID=unique_id, idx=i) @@ -2036,40 +2279,67 @@ def addPollLoop(self, input_data_path, last_time_index): proc_params = {} proc_params["exp_id"] = expID proc_params["desc"] = tableDescToolTip - proc_params["config_path"] = str(Path(config_path).absolute()) - proc_params["input_path"] = str(Path(input_dir).absolute()) - proc_params["output_path"] = str(Path(output_dir).absolute()) - proc_params["output_path_parent"] = str(Path(output_dir).parent.absolute()) + proc_params["config_path"] = str( + Path(config_path).absolute() + ) + proc_params["input_path"] = str( + Path(input_dir).absolute() + ) + proc_params["output_path"] = str( + Path(output_dir).absolute() + ) + proc_params["output_path_parent"] = str( + Path(output_dir).parent.absolute() + ) proc_params["show"] = False proc_params["rx"] = 1 - - tableEntryWorker1 = AddTableEntryWorkerThread(tableID, tableDescToolTip, proc_params) - tableEntryWorker1.add_tableentry_signal.connect(self.addTableEntry) + + tableEntryWorker1 = AddTableEntryWorkerThread( + tableID, tableDescToolTip, proc_params + ) + tableEntryWorker1.add_tableentry_signal.connect( + self.addTableEntry + ) tableEntryWorker1.start() - if json.dumps(sorted_dict_acq) == json.dumps(sorted_dict_final) and _breakFlag: - - tableEntryWorker2 = AddOTFTableEntryWorkerThread(input_data_path, False, False) - tableEntryWorker2.add_tableOTFentry_signal.connect(self.addRemoveCheckOTFTableEntry) + if ( + json.dumps(sorted_dict_acq) + == json.dumps(sorted_dict_final) + and _breakFlag + ): + + tableEntryWorker2 = AddOTFTableEntryWorkerThread( + input_data_path, False, False + ) + tableEntryWorker2.add_tableOTFentry_signal.connect( + self.add_remove_check_OTF_table_entry + ) tableEntryWorker2.start() # let child threads finish their work before exiting the parent thread - while tableEntryWorker1.isRunning() or tableEntryWorker2.isRunning(): + while ( + tableEntryWorker1.isRunning() + or tableEntryWorker2.isRunning() + ): time.sleep(1) time.sleep(5) break - - last_time_index = my_dict_time_indices_curr-2 + + last_time_index = my_dict_time_indices_curr - 2 except Exception as exc: print(exc.args) - print("Exiting polling for dataset: {data_path}".format(data_path=input_data_path)) + print( + "Exiting polling for dataset: {data_path}".format( + data_path=input_data_path + ) + ) break - def loadZattrsDirectlyAsDict(self, zattrsFilePathDir): + def load_zattrs_directly_as_dict(self, zattrsFilePathDir): try: file_path = os.path.join(zattrsFilePathDir, ".zattrs") f = open(file_path, "r") - txt = f.read() + txt = f.read() f.close() return json.loads(txt) except Exception as exc: @@ -2351,7 +2621,9 @@ def add_pydantic_to_container( ) elif isinstance(new_widget, widgets.FileEdit): if len(json_dict[field]) > 0: - extension = os.path.splitext(json_dict[field])[1] + extension = os.path.splitext(json_dict[field])[ + 1 + ] if len(extension) > 0: new_widget.value = Path( json_dict[field] @@ -2403,19 +2675,29 @@ def get_pydantic_kwargs( # copied from main_widget # file open/select dialog - def _open_file_dialog(self, default_path, type, filter="All Files (*)"): + def open_file_dialog(self, default_path, type, filter="All Files (*)"): if type == "dir": - return self._open_dialog("select a directory", str(default_path), type, filter) + return self.open_dialog( + "select a directory", str(default_path), type, filter + ) elif type == "file": - return self._open_dialog("select a file", str(default_path), type, filter) + return self.open_dialog( + "select a file", str(default_path), type, filter + ) elif type == "files": - return self._open_dialog("select file(s)", str(default_path), type, filter) + return self.open_dialog( + "select file(s)", str(default_path), type, filter + ) elif type == "save": - return self._open_dialog("save a file", str(default_path), type, filter) + return self.open_dialog( + "save a file", str(default_path), type, filter + ) else: - return self._open_dialog("select a directory", str(default_path), type, filter) + return self.open_dialog( + "select a directory", str(default_path), type, filter + ) - def _open_dialog(self, title, ref, type, filter="All Files (*)"): + def open_dialog(self, title, ref, type, filter="All Files (*)"): """ opens pop-up dialogue for the user to choose a specific file or directory. @@ -2452,6 +2734,7 @@ def _open_dialog(self, title, ref, type, filter="All Files (*)"): return path + class MyWorker: def __init__(self, formLayout, tab_recon: Ui_ReconTab_Form, parentForm): @@ -2466,30 +2749,34 @@ def __init__(self, formLayout, tab_recon: Ui_ReconTab_Form, parentForm): self.pool = None self.futures = [] # https://click.palletsprojects.com/en/stable/testing/ - # self.runner = CliRunner() + # self.runner = CliRunner() # jobs_mgmt.shared_var_jobs = self.JobsManager.shared_var_jobs self.JobsMgmt = jobs_mgmt.JobsManagement() self.useServer = True self.serverRunning = True - self.server_socket = None - self.isInitialized = False + self.server_socket = None + self.isInitialized = False - def initialize(self): - if not self.isInitialized: - thread = threading.Thread(target=self.startServer) + def initialize(self): + if not self.isInitialized: + thread = threading.Thread(target=self.start_server) thread.start() - self.workerThreadRowDeletion = RowDeletionWorkerThread(self.formLayout) - self.workerThreadRowDeletion.removeRowSignal.connect(self.tab_recon.removeRow) + self.workerThreadRowDeletion = RowDeletionWorkerThread( + self.formLayout + ) + self.workerThreadRowDeletion.removeRowSignal.connect( + self.tab_recon.remove_row + ) self.workerThreadRowDeletion.start() self.isInitialized = True - def setNewInstances(self, formLayout, tab_recon, parentForm): + def set_new_instances(self, formLayout, tab_recon, parentForm): self.formLayout: QFormLayout = formLayout self.tab_recon: Ui_ReconTab_Form = tab_recon self.ui: QWidget = parentForm - self.workerThreadRowDeletion.setNewInstances(formLayout) + self.workerThreadRowDeletion.set_new_instances(formLayout) - def findWidgetRowInLayout(self, strID): + def find_widget_row_in_layout(self, strID): layout: QFormLayout = self.formLayout for idx in range(0, layout.rowCount()): widgetItem = layout.itemAt(idx) @@ -2500,7 +2787,7 @@ def findWidgetRowInLayout(self, strID): return idx return -1 - def startServer(self): + def start_server(self): try: if not self.useServer: return @@ -2517,9 +2804,12 @@ def startServer(self): client_socket, address = self.server_socket.accept() if self.ui is not None and not self.ui.isVisible(): break - try: + try: # dont block the server thread - thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=("", "", "", "", client_socket),) + thread = threading.Thread( + target=self.decode_client_data, + args=("", "", "", "", client_socket), + ) thread.start() except Exception as exc: print(exc.args) @@ -2532,7 +2822,7 @@ def startServer(self): return # ignore - will cause an exception on napari close but that is fine and does the job print(exc.args) - def stopServer(self): + def stop_server(self): try: if self.server_socket is not None: self.serverRunning = False @@ -2540,39 +2830,47 @@ def stopServer(self): except Exception as exc: print(exc.args) - def getMaxCPU_cores(self): + def get_max_CPU_cores(self): return self.max_cores - def setPoolThreads(self, t): + def set_pool_threads(self, t): if t > 0 and t < self.max_cores: self.threadPool = t - def startPool(self): + def start_pool(self): if self.pool is None: - self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=self.threadPool) + self.pool = concurrent.futures.ThreadPoolExecutor( + max_workers=self.threadPool + ) - def shutDownPool(self): + def shut_down_pool(self): self.pool.shutdown(wait=True) - # the table update thread can be called from multiple points/threads - # on errors - table row item is updated but there is no row deletion - # on successful processing - the row item is expected to be deleted - # row is being deleted from a seperate thread for which we need to connect using signal - def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_folder_path="", client_socket=None): - # finished will be updated by the job - submitit status - jobIdx = str(jobIdx) - if client_socket is not None and expIdx=="" and jobIdx=="": + # This method handles each client response thread. It parses the information received from the client + # and is responsible for parsing each well/pos Job if the case may be and starting individual update threads + # using the tableUpdateAndCleaupThread() method + # This is also handling an unused "CoNvErTeR" functioning that can be implemented on 3rd party apps + def decode_client_data(self, + expIdx="", + jobIdx="", + wellName="", + logs_folder_path="", + client_socket=None,): + + if client_socket is not None and expIdx == "" and jobIdx == "": try: buf = client_socket.recv(10240) if len(buf) > 0: if b"\n" in buf: dataList = buf.split(b"\n") else: - dataList = [buf] + dataList = [buf] for data in dataList: - if len(data)>0: + if len(data) > 0: decoded_string = data.decode() - if "CoNvErTeR" in decoded_string: # this request came from an agnostic route - requires processing + if ( + "CoNvErTeR" in decoded_string + ): # this request came from an agnostic route - requires processing json_str = str(decoded_string) json_obj = json.loads(json_str) converter_params = json_obj["CoNvErTeR"] @@ -2588,99 +2886,183 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol proc_params = {} proc_params["exp_id"] = expID - proc_params["desc"] = expID + proc_params["desc"] = expID proc_params["input_path"] = str(input_data) proc_params["output_path"] = str(output_data) - proc_params["output_path_parent"] = str(Path(output_data).parent.absolute()) + proc_params["output_path_parent"] = str( + Path(output_data).parent.absolute() + ) proc_params["show"] = False proc_params["rx"] = 1 if config_path == "": model = None - if len(self.tab_recon.pydantic_classes) > 0: - for item in self.tab_recon.pydantic_classes: + if ( + len(self.tab_recon.pydantic_classes) + > 0 + ): + for ( + item + ) in self.tab_recon.pydantic_classes: if mode == item["selected_modes"]: cls = item["class"] - cls_container = item["container"] - exclude_modes = item["exclude_modes"] - output_LineEdit = item["output_LineEdit"] - output_parent_dir = item["output_parent_dir"] - full_out_path = os.path.join(output_parent_dir, output_LineEdit.value) + cls_container = item[ + "container" + ] + exclude_modes = item[ + "exclude_modes" + ] + output_LineEdit = item[ + "output_LineEdit" + ] + output_parent_dir = item[ + "output_parent_dir" + ] + full_out_path = os.path.join( + output_parent_dir, + output_LineEdit.value, + ) # gather input/out locations output_dir = full_out_path if output_data == "": output_data = output_dir - proc_params["output_path"] = str(output_data) + proc_params[ + "output_path" + ] = str(output_data) # build up the arguments for the pydantic model given the current container if cls is None: - self.tab_recon.messageBox("No model defined !") + self.tab_recon.message_box( + "No model defined !" + ) return pydantic_kwargs = {} - pydantic_kwargs, ret_msg = self.tab_recon.get_and_validate_pydantic_args( - cls_container, cls, pydantic_kwargs, exclude_modes + pydantic_kwargs, ret_msg = ( + self.tab_recon.get_and_validate_pydantic_args( + cls_container, + cls, + pydantic_kwargs, + exclude_modes, + ) ) if pydantic_kwargs is None: - self.tab_recon.messageBox(ret_msg) + self.tab_recon.message_box( + ret_msg + ) return - input_channel_names, ret_msg = self.tab_recon.clean_string_for_list( - "input_channel_names", pydantic_kwargs["input_channel_names"] + ( + input_channel_names, + ret_msg, + ) = self.tab_recon.clean_string_for_list( + "input_channel_names", + pydantic_kwargs[ + "input_channel_names" + ], ) if input_channel_names is None: - self.tab_recon.messageBox(ret_msg) + self.tab_recon.message_box( + ret_msg + ) return - pydantic_kwargs["input_channel_names"] = input_channel_names - - time_indices, ret_msg = self.tab_recon.clean_string_int_for_list( - "time_indices", pydantic_kwargs["time_indices"] + pydantic_kwargs[ + "input_channel_names" + ] = input_channel_names + + time_indices, ret_msg = ( + self.tab_recon.clean_string_int_for_list( + "time_indices", + pydantic_kwargs[ + "time_indices" + ], + ) ) if time_indices is None: - self.tab_recon.messageBox(ret_msg) + self.tab_recon.message_box( + ret_msg + ) return - pydantic_kwargs["time_indices"] = time_indices - - time_indices, ret_msg = self.tab_recon.clean_string_int_for_list( - "time_indices", pydantic_kwargs["time_indices"] + pydantic_kwargs[ + "time_indices" + ] = time_indices + + time_indices, ret_msg = ( + self.tab_recon.clean_string_int_for_list( + "time_indices", + pydantic_kwargs[ + "time_indices" + ], + ) ) if time_indices is None: - self.tab_recon.messageBox(ret_msg) + self.tab_recon.message_box( + ret_msg + ) return - pydantic_kwargs["time_indices"] = time_indices - - if "birefringence" in pydantic_kwargs.keys(): - background_path, ret_msg = self.tab_recon.clean_path_string_when_empty( + pydantic_kwargs[ + "time_indices" + ] = time_indices + + if ( + "birefringence" + in pydantic_kwargs.keys() + ): + ( + background_path, + ret_msg, + ) = self.tab_recon.clean_path_string_when_empty( "background_path", - pydantic_kwargs["birefringence"]["apply_inverse"][ + pydantic_kwargs[ + "birefringence" + ]["apply_inverse"][ "background_path" ], ) if background_path is None: - self.tab_recon.messageBox(ret_msg) + self.tab_recon.message_box( + ret_msg + ) return - pydantic_kwargs["birefringence"]["apply_inverse"][ + pydantic_kwargs[ + "birefringence" + ]["apply_inverse"][ "background_path" ] = background_path # validate and return errors if None - pydantic_model, ret_msg = self.tab_recon.validate_pydantic_model( - cls, pydantic_kwargs + pydantic_model, ret_msg = ( + self.tab_recon.validate_pydantic_model( + cls, pydantic_kwargs + ) ) if pydantic_model is None: - self.tab_recon.messageBox(ret_msg) + self.tab_recon.message_box( + ret_msg + ) return model = pydantic_model break if model is None: - model, msg = self.tab_recon.buildModel(mode) - yaml_path = os.path.join(str(Path(output_data).parent.absolute()), expID+".yml") + model, msg = self.tab_recon.build_model( + mode + ) + yaml_path = os.path.join( + str( + Path(output_data).parent.absolute() + ), + expID + ".yml", + ) utils.model_to_yaml(model, yaml_path) proc_params["config_path"] = str(yaml_path) - tableEntryWorker = AddTableEntryWorkerThread(expID, expID, proc_params) - tableEntryWorker.add_tableentry_signal.connect(self.tab_recon.addTableEntry) + tableEntryWorker = AddTableEntryWorkerThread( + expID, expID, proc_params + ) + tableEntryWorker.add_tableentry_signal.connect( + self.tab_recon.addTableEntry + ) tableEntryWorker.start() time.sleep(10) return @@ -2692,34 +3074,82 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol jobIdx = json_obj[k]["jID"] wellName = json_obj[k]["pos"] logs_folder_path = json_obj[k]["log"] - if expIdx not in self.results.keys(): # this job came from agnostic CLI route - no processing + if ( + expIdx not in self.results.keys() + ): # this job came from agnostic CLI route - no processing now = datetime.datetime.now() ms = now.strftime("%f")[:3] - unique_id = now.strftime("%Y_%m_%d_%H_%M_%S_") + ms - expIdx = expIdx +"-"+ unique_id - self.JobsMgmt.putJobInList(None, expIdx, str(jobIdx), wellName, mode="server") + unique_id = ( + now.strftime("%Y_%m_%d_%H_%M_%S_") + ms + ) + expIdx = expIdx + "-" + unique_id + self.JobsMgmt.put_Job_in_list( + None, + expIdx, + str(jobIdx), + wellName, + mode="server", + ) # print("Submitting Job: {job} expIdx: {expIdx}".format(job=jobIdx, expIdx=expIdx)) - thread = threading.Thread(target=self.tableUpdateAndCleaupThread,args=(expIdx, jobIdx, wellName, logs_folder_path, client_socket)) + thread = threading.Thread( + target=self.table_update_and_cleaup_thread, + args=( + expIdx, + jobIdx, + wellName, + logs_folder_path, + client_socket, + ), + ) thread.start() return except Exception as exc: print(exc.args) + # the table update thread can be called from multiple points/threads + # on errors - table row item is updated but there is no row deletion + # on successful processing - the row item is expected to be deleted + # row is being deleted from a seperate thread for which we need to connect using signal + + # This is handling essentially each job thread. Points of entry are on a failed job submission + # which then calls this to update based on the expID (used for .yml naming). On successful job + # submissions jobID, the point of entry is via the socket connection the GUI is listening and + # then spawns a new thread to avoid blocking of other connections. + # If a job submission spawns more jobs then this also calls other methods via signal to create + # the required GUI components in the main thread. + # Once we have expID and jobID this thread periodically loops and updates each job status and/or + # the job error by reading the log files. Using certain keywords + # eg JOB_COMPLETION_STR = "Job completed successfully" we determine the progress. We also create + # a map for expID which might have multiple jobs to determine when a reconstruction is + # finished vs a single job finishing. + # The loop ends based on user, time-out, job(s) completion and errors and handles removal of + # processing GUI table items (on main thread). + # Based on the conditions the loop will end calling clientRelease() + def table_update_and_cleaup_thread( + self, + expIdx="", + jobIdx="", + wellName="", + logs_folder_path="", + client_socket=None, + ): + jobIdx = str(jobIdx) + # ToDo: Another approach to this could be to implement a status thread on the client side # Since the client is already running till the job is completed, the client could ping status # at regular intervals and also provide results and exceptions we currently read from the file # Currently we only send JobID/UniqueID pair from Client to Server. This would reduce multiple threads # server side. - # For row removal use a Queued list approach for better stability if expIdx != "" and jobIdx != "": - # this request came from server so we can wait for the Job to finish and update progress - # some wait logic needs to be added otherwise for unknown errors this thread will persist - # perhaps set a time out limit and then update the status window and then exit - - if expIdx not in self.results.keys(): # this job came from agnostic route + # this request came from server listening so we wait for the Job to finish and update progress + if ( + expIdx not in self.results.keys() + ): proc_params = {} - tableID = "{exp} - {job} ({pos})".format(exp=expIdx, job=jobIdx, pos=wellName) + tableID = "{exp} - {job} ({pos})".format( + exp=expIdx, job=jobIdx, pos=wellName + ) proc_params["exp_id"] = expIdx proc_params["desc"] = tableID proc_params["config_path"] = "" @@ -2728,11 +3158,15 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol proc_params["output_path_parent"] = "" proc_params["show"] = False proc_params["rx"] = 1 - - tableEntryWorker = AddTableEntryWorkerThread(tableID, tableID, proc_params) - tableEntryWorker.add_tableentry_signal.connect(self.tab_recon.addTableEntry) + + tableEntryWorker = AddTableEntryWorkerThread( + tableID, tableID, proc_params + ) + tableEntryWorker.add_tableentry_signal.connect( + self.tab_recon.addTableEntry + ) tableEntryWorker.start() - + while expIdx not in self.results.keys(): time.sleep(1) @@ -2741,31 +3175,41 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol else: params = self.results[expIdx]["JobUNK"].copy() - if jobIdx not in self.results[expIdx].keys() and len(self.results[expIdx].keys()) == 1: + if ( + jobIdx not in self.results[expIdx].keys() + and len(self.results[expIdx].keys()) == 1 + ): # this is the first job params["primary"] = True self.results[expIdx][jobIdx] = params - elif jobIdx not in self.results[expIdx].keys() and len(self.results[expIdx].keys()) > 1: + elif ( + jobIdx not in self.results[expIdx].keys() + and len(self.results[expIdx].keys()) > 1 + ): # this is a new job # we need to create cancel and job status windows and add to parent container params["primary"] = False - NEW_WIDGETS_QUEUE.append(expIdx+jobIdx) - parentLayout:QVBoxLayout = params["parent_layout"] - worker_thread = AddWidgetWorkerThread(parentLayout, expIdx, jobIdx, params["desc"], wellName) - worker_thread.add_widget_signal.connect(self.tab_recon.add_widget) + NEW_WIDGETS_QUEUE.append(expIdx + jobIdx) + parentLayout: QVBoxLayout = params["parent_layout"] + worker_thread = AddWidgetWorkerThread( + parentLayout, expIdx, jobIdx, params["desc"], wellName + ) + worker_thread.add_widget_signal.connect( + self.tab_recon.add_widget + ) NEW_WIDGETS_QUEUE_THREADS.append(worker_thread) - while (len(NEW_WIDGETS_QUEUE_THREADS) > 0): + while len(NEW_WIDGETS_QUEUE_THREADS) > 0: s_worker_thread = NEW_WIDGETS_QUEUE_THREADS.pop(0) s_worker_thread.start() time.sleep(1) # wait for new components reference - while (expIdx+jobIdx in NEW_WIDGETS_QUEUE): + while expIdx + jobIdx in NEW_WIDGETS_QUEUE: time.sleep(1) - _cancelJobBtn = MULTI_JOBS_REFS[expIdx+jobIdx]["cancelBtn"] - _infoBox = MULTI_JOBS_REFS[expIdx+jobIdx]["infobox"] + _cancelJobBtn = MULTI_JOBS_REFS[expIdx + jobIdx]["cancelBtn"] + _infoBox = MULTI_JOBS_REFS[expIdx + jobIdx]["infobox"] params["table_entry_infoBox"] = _infoBox params["cancelJobButton"] = _cancelJobBtn @@ -2776,9 +3220,11 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol _txtForInfoBox = "Updating {id}-{pos}: Please wait... \nJobID assigned: {jID} ".format( id=params["desc"], pos=wellName, jID=jobIdx - ) + ) try: - _cancelJobBtn.text = "Cancel Job {jID} ({posName})".format(jID=jobIdx, posName=wellName) + _cancelJobBtn.text = "Cancel Job {jID} ({posName})".format( + jID=jobIdx, posName=wellName + ) _cancelJobBtn.enabled = True _infoBox.setText(_txtForInfoBox) except: @@ -2793,46 +3239,62 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol jobTXT = "" # print("Updating Job: {job} expIdx: {expIdx}".format(job=jobIdx, expIdx=expIdx)) while True: - time.sleep(1) # update every sec and exit on break + time.sleep(1) # update every sec and exit on break try: if "cancel called" in _cancelJobBtn.text: - json_obj = {"uID":expIdx, "jID":jobIdx, "command":"cancel"} - json_str = json.dumps(json_obj)+"\n" + json_obj = { + "uID": expIdx, + "jID": jobIdx, + "command": "cancel", + } + json_str = json.dumps(json_obj) + "\n" client_socket.send(json_str.encode()) params["status"] = STATUS_user_cancelled_job _infoBox.setText( - "User called for Cancel Job Request\n" - + "Please check terminal output for Job status..\n\n" - + jobTXT - ) - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=1) - break # cancel called by user + "User called for Cancel Job Request\n" + + "Please check terminal output for Job status..\n\n" + + jobTXT + ) + self.client_release( + expIdx, jobIdx, client_socket, params, reason=1 + ) + break # cancel called by user if _infoBox == None: params["status"] = STATUS_user_cleared_job - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=2) + self.client_release( + expIdx, jobIdx, client_socket, params, reason=2 + ) break # deleted by user - no longer needs updating if _infoBox: pass except Exception as exc: print(exc.args) params["status"] = STATUS_user_cleared_job - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=3) + self.client_release( + expIdx, jobIdx, client_socket, params, reason=3 + ) break # deleted by user - no longer needs updating - if self.JobsMgmt.hasSubmittedJob(expIdx, jobIdx, mode="server"): + if self.JobsMgmt.has_submitted_job( + expIdx, jobIdx, mode="server" + ): if params["status"] in [STATUS_finished_job]: - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=4) + self.client_release( + expIdx, jobIdx, client_socket, params, reason=4 + ) break elif params["status"] in [STATUS_errored_job]: - jobERR = self.JobsMgmt.checkForJobIDFile( + jobERR = self.JobsMgmt.check_for_jobID_File( jobIdx, logs_folder_path, extension="err" ) _infoBox.setText( jobIdx + "\n" + params["desc"] + "\n\n" + jobERR ) - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=5) + self.client_release( + expIdx, jobIdx, client_socket, params, reason=5 + ) break else: - jobTXT = self.JobsMgmt.checkForJobIDFile( + jobTXT = self.JobsMgmt.check_for_jobID_File( jobIdx, logs_folder_path, extension="out" ) try: @@ -2840,9 +3302,13 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol # print(jobIdx + " not started yet") time.sleep(2) _tUpdateCount += 2 - if _tUpdateCount > 10: # if out file is empty for 10s, check the err file to update user - jobERR = self.JobsMgmt.checkForJobIDFile( - jobIdx, logs_folder_path, extension="err" + if ( + _tUpdateCount > 10 + ): # if out file is empty for 10s, check the err file to update user + jobERR = self.JobsMgmt.check_for_jobID_File( + jobIdx, + logs_folder_path, + extension="err", ) _infoBox.setText( jobIdx @@ -2852,17 +3318,26 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + jobERR ) if _tUpdateCount > _tUpdateCountTimeout: - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=6) + self.client_release( + expIdx, + jobIdx, + client_socket, + params, + reason=6, + ) break - elif ( - params["status"] - == STATUS_finished_job - ): - rowIdx = self.findWidgetRowInLayout(expIdx) + elif params["status"] == STATUS_finished_job: + rowIdx = self.find_widget_row_in_layout(expIdx) # check to ensure row deletion due to shrinking table # if not deleted try to delete again if rowIdx < 0: - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=7) + self.client_release( + expIdx, + jobIdx, + client_socket, + params, + reason=7, + ) break else: break @@ -2876,7 +3351,7 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol # break - based on status elif JOB_TRIGGERED_EXC in jobTXT: params["status"] = STATUS_errored_job - jobERR = self.JobsMgmt.checkForJobIDFile( + jobERR = self.JobsMgmt.check_for_jobID_File( jobIdx, logs_folder_path, extension="err" ) _infoBox.setText( @@ -2888,7 +3363,13 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + "\n\n" + jobERR ) - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=8) + self.client_release( + expIdx, + jobIdx, + client_socket, + params, + reason=8, + ) break elif JOB_RUNNING_STR in jobTXT: params["status"] = STATUS_running_job @@ -2905,10 +3386,16 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + jobTXT ) if _tUpdateCount > _tUpdateCountTimeout: - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=9) + self.client_release( + expIdx, + jobIdx, + client_socket, + params, + reason=9, + ) break else: - jobERR = self.JobsMgmt.checkForJobIDFile( + jobERR = self.JobsMgmt.check_for_jobID_File( jobIdx, logs_folder_path, extension="err" ) _infoBox.setText( @@ -2918,17 +3405,25 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol + "\n\n" + jobERR ) - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=10) + self.client_release( + expIdx, + jobIdx, + client_socket, + params, + reason=10, + ) break except Exception as exc: print(exc.args) else: - self.clientRelease(expIdx, jobIdx, client_socket, params, reason=11) + self.client_release( + expIdx, jobIdx, client_socket, params, reason=11 + ) break else: # this would occur when an exception happens on the pool side before or during job submission # we dont have a job ID and will update based on exp_ID/uID - # if job submission was not successful we can assume the client is not listening + # if job submission was not successful we can assume the client is not listening # and does not require a clientRelease cmd for uID in self.results.keys(): params = self.results[uID]["JobUNK"] @@ -2937,27 +3432,36 @@ def tableUpdateAndCleaupThread(self, expIdx="", jobIdx="", wellName="", logs_fol poolERR = params["error"] _infoBox.setText(poolERR) - def clientRelease(self, expIdx, jobIdx, client_socket, params, reason=0): + def client_release(self, expIdx, jobIdx, client_socket, params, reason=0): # only need to release client from primary job # print("clientRelease Job: {job} expIdx: {expIdx} reason:{reason}".format(job=jobIdx, expIdx=expIdx, reason=reason)) - self.JobsMgmt.putJobCompletionInList(True, expIdx, jobIdx) + self.JobsMgmt.put_Job_completion_in_list(True, expIdx, jobIdx) showData_thread = None if params["primary"]: if "show" in params: if params["show"]: # Read reconstruction data - showData_thread = ShowDataWorkerThread(params["output_path"]) - showData_thread.show_data_signal.connect(self.tab_recon.showDataset) + showData_thread = ShowDataWorkerThread( + params["output_path"] + ) + showData_thread.show_data_signal.connect( + self.tab_recon.show_dataset + ) showData_thread.start() - while not self.JobsMgmt.checkAllExpJobsCompletion(expIdx): + # for multi-job expID we need to check completion for all of them + while not self.JobsMgmt.check_all_ExpJobs_completion(expIdx): time.sleep(1) - json_obj = {"uID":expIdx, "jID":jobIdx,"command": "clientRelease"} - json_str = json.dumps(json_obj)+"\n" + json_obj = { + "uID": expIdx, + "jID": jobIdx, + "command": "clientRelease", + } + json_str = json.dumps(json_obj) + "\n" client_socket.send(json_str.encode()) - ROW_POP_QUEUE.append(expIdx) - # print("FINISHED") + ROW_POP_QUEUE.append(expIdx) + # print("FINISHED") if self.pool is not None: if self.pool._work_queue.qsize() == 0: @@ -2968,14 +3472,16 @@ def clientRelease(self, expIdx, jobIdx, client_socket, params, reason=0): while showData_thread.isRunning(): time.sleep(3) - def runInPool(self, params): + def run_in_pool(self, params): if not self.isInitialized: self.initialize() - self.startPool() + self.start_pool() self.results[params["exp_id"]] = {} self.results[params["exp_id"]]["JobUNK"] = params - self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_running_pool + self.results[params["exp_id"]]["JobUNK"][ + "status" + ] = STATUS_running_pool self.results[params["exp_id"]]["JobUNK"]["error"] = "" try: @@ -2986,31 +3492,39 @@ def runInPool(self, params): f = self.pool.submit(self.run, params) self.futures.append(f) except Exception as exc: - self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) - self.tableUpdateAndCleaupThread() + self.results[params["exp_id"]]["JobUNK"][ + "status" + ] = STATUS_errored_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = str( + "\n".join(exc.args) + ) + self.table_update_and_cleaup_thread() - def runMultiInPool(self, multi_params_as_list): - self.startPool() + def run_multi_in_pool(self, multi_params_as_list): + self.start_pool() for params in multi_params_as_list: self.results[params["exp_id"]] = {} self.results[params["exp_id"]]["JobUNK"] = params - self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_submitted_pool + self.results[params["exp_id"]]["JobUNK"][ + "status" + ] = STATUS_submitted_pool self.results[params["exp_id"]]["JobUNK"]["error"] = "" try: self.pool.map(self.run, multi_params_as_list) except Exception as exc: for params in multi_params_as_list: - self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool + self.results[params["exp_id"]]["JobUNK"][ + "status" + ] = STATUS_errored_pool self.results[params["exp_id"]]["JobUNK"]["error"] = str( "\n".join(exc.args) ) - self.tableUpdateAndCleaupThread() + self.table_update_and_cleaup_thread() - def getResults(self): + def get_results(self): return self.results - def getResult(self, exp_id): + def get_result(self, exp_id): return self.results[exp_id] def run(self, params): @@ -3020,21 +3534,27 @@ def run(self, params): self.results[params["exp_id"]] = {} self.results[params["exp_id"]]["JobUNK"] = params self.results[params["exp_id"]]["JobUNK"]["error"] = "" - self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_running_pool + self.results[params["exp_id"]]["JobUNK"][ + "status" + ] = STATUS_running_pool try: # does need further threading ? probably not ! thread = threading.Thread( - target=self.runInSubProcess, args=(params,) + target=self.run_in_subprocess, args=(params,) ) thread.start() except Exception as exc: - self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) - self.tableUpdateAndCleaupThread() + self.results[params["exp_id"]]["JobUNK"][ + "status" + ] = STATUS_errored_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = str( + "\n".join(exc.args) + ) + self.table_update_and_cleaup_thread() - def runInSubProcess(self, params): + def run_in_subprocess(self, params): """function that initiates the processing on the CLI""" try: input_path = str(params["input_path"]) @@ -3044,7 +3564,9 @@ def runInSubProcess(self, params): rx = str(params["rx"]) mainfp = str(jobs_mgmt.FILE_PATH) - self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_submitted_job + self.results[params["exp_id"]]["JobUNK"][ + "status" + ] = STATUS_submitted_job proc = subprocess.run( [ @@ -3060,7 +3582,7 @@ def runInSubProcess(self, params): "-rx", str(rx), "-uid", - uid + uid, ] ) self.results[params["exp_id"]]["JobUNK"]["proc"] = proc @@ -3070,26 +3592,33 @@ def runInSubProcess(self, params): ) except Exception as exc: - self.results[params["exp_id"]]["JobUNK"]["status"] = STATUS_errored_pool - self.results[params["exp_id"]]["JobUNK"]["error"] = str("\n".join(exc.args)) - self.tableUpdateAndCleaupThread() - + self.results[params["exp_id"]]["JobUNK"][ + "status" + ] = STATUS_errored_pool + self.results[params["exp_id"]]["JobUNK"]["error"] = str( + "\n".join(exc.args) + ) + self.table_update_and_cleaup_thread() + class ShowDataWorkerThread(QThread): """Worker thread for sending signal for adding component when request comes from a different thread""" + show_data_signal = pyqtSignal(str) def __init__(self, path): super().__init__() self.path = path - def run(self): + def run(self): # Emit the signal to add the widget to the main thread self.show_data_signal.emit(self.path) + class AddOTFTableEntryWorkerThread(QThread): """Worker thread for sending signal for adding component when request comes from a different thread""" + add_tableOTFentry_signal = pyqtSignal(str, bool, bool) def __init__(self, OTF_dir_path, bool_msg, doCheck=False): @@ -3098,12 +3627,16 @@ def __init__(self, OTF_dir_path, bool_msg, doCheck=False): self.bool_msg = bool_msg self.doCheck = doCheck - def run(self): + def run(self): # Emit the signal to add the widget to the main thread - self.add_tableOTFentry_signal.emit(self.OTF_dir_path, self.bool_msg, self.doCheck) + self.add_tableOTFentry_signal.emit( + self.OTF_dir_path, self.bool_msg, self.doCheck + ) + class AddTableEntryWorkerThread(QThread): """Worker thread for sending signal for adding component when request comes from a different thread""" + add_tableentry_signal = pyqtSignal(str, str, dict) def __init__(self, expID, desc, params): @@ -3112,12 +3645,14 @@ def __init__(self, expID, desc, params): self.desc = desc self.params = params - def run(self): + def run(self): # Emit the signal to add the widget to the main thread self.add_tableentry_signal.emit(self.expID, self.desc, self.params) + class AddWidgetWorkerThread(QThread): """Worker thread for sending signal for adding component when request comes from a different thread""" + add_widget_signal = pyqtSignal(QVBoxLayout, str, str, str, str) def __init__(self, layout, expID, jID, desc, wellName): @@ -3128,11 +3663,11 @@ def __init__(self, layout, expID, jID, desc, wellName): self.desc = desc self.wellName = wellName - def run(self): + def run(self): # Emit the signal to add the widget to the main thread - self.add_widget_signal.emit(self.layout, self.expID, self.jID, self.desc, self.wellName) - -ROW_POP_QUEUE = [] + self.add_widget_signal.emit( + self.layout, self.expID, self.jID, self.desc, self.wellName + ) class RowDeletionWorkerThread(QThread): """Searches for a row based on its ID and then @@ -3144,12 +3679,12 @@ def __init__(self, formLayout): super().__init__() self.formLayout = formLayout - def setNewInstances(self, formLayout): + def set_new_instances(self, formLayout): self.formLayout: QFormLayout = formLayout # we might deal with race conditions with a shrinking table # find out widget and return its index - def findWidgetRowInLayout(self, strID): + def find_widget_row_in_layout(self, strID): layout: QFormLayout = self.formLayout for idx in range(0, layout.rowCount()): widgetItem = layout.itemAt(idx) @@ -3166,7 +3701,7 @@ def run(self): if len(ROW_POP_QUEUE) > 0: stringID = ROW_POP_QUEUE.pop(0) # Emit the signal to remove the row - deleteRow = self.findWidgetRowInLayout(stringID) + deleteRow = self.find_widget_row_in_layout(stringID) if deleteRow > -1: self.removeRowSignal.emit(int(deleteRow), str(stringID)) time.sleep(1) @@ -3175,7 +3710,8 @@ def run(self): class DropButton(QPushButton): """A drag & drop PushButton to load model file(s)""" - def __init__(self, text, parent=None, recon_tab:Ui_ReconTab_Form=None): + + def __init__(self, text, parent=None, recon_tab: Ui_ReconTab_Form = None): super().__init__(text, parent) self.setAcceptDrops(True) self.recon_tab = recon_tab @@ -3189,11 +3725,12 @@ def dropEvent(self, event): for url in event.mimeData().urls(): filepath = url.toLocalFile() files.append(filepath) - self.recon_tab.openModelFiles(files) + self.recon_tab.open_model_files(files) class DropWidget(QWidget): - """A drag & drop widget container to load model file(s) """ - def __init__(self, recon_tab:Ui_ReconTab_Form=None): + """A drag & drop widget container to load model file(s)""" + + def __init__(self, recon_tab: Ui_ReconTab_Form = None): super().__init__() self.setAcceptDrops(True) self.recon_tab = recon_tab @@ -3207,10 +3744,11 @@ def dropEvent(self, event): for url in event.mimeData().urls(): filepath = url.toLocalFile() files.append(filepath) - self.recon_tab.openModelFiles(files) + self.recon_tab.open_model_files(files) class ScrollableLabel(QScrollArea): - """A scrollable label widget used for Job entry """ + """A scrollable label widget used for Job entry""" + def __init__(self, text, *args, **kwargs): super().__init__(*args, **kwargs) @@ -3250,6 +3788,7 @@ def __init__(self): def resizeEvent(self, event): self.resized.emit() super().resizeEvent(event) + class CollapsibleBox(QWidget): """A collapsible widget""" @@ -3298,7 +3837,9 @@ def setNewName(self, name): def on_pressed(self): checked = self.toggle_button.isChecked() self.toggle_button.setArrowType( - QtCore.Qt.ArrowType.DownArrow if not checked else QtCore.Qt.ArrowType.RightArrow + QtCore.Qt.ArrowType.DownArrow + if not checked + else QtCore.Qt.ArrowType.RightArrow ) self.toggle_animation.setDirection( QtCore.QAbstractAnimation.Direction.Forward @@ -3331,10 +3872,9 @@ def setContentLayout(self, layout): content_animation.setStartValue(0) content_animation.setEndValue(content_height) - # VScode debugging if __name__ == "__main__": import napari napari.Viewer() - napari.run() \ No newline at end of file + napari.run() diff --git a/recOrder/scripts/simulate_zarr_acq.py b/recOrder/scripts/simulate_zarr_acq.py index b13088da..67b0fe09 100644 --- a/recOrder/scripts/simulate_zarr_acq.py +++ b/recOrder/scripts/simulate_zarr_acq.py @@ -2,8 +2,9 @@ from iohub.convert import TIFFConverter from iohub.ngff import open_ome_zarr from recOrder.cli.utils import create_empty_hcs_zarr +from recOrder.cli import jobs_mgmt -import time, threading, os, shutil, json +import time, threading, os, shutil, subprocess # This script is a demo .zarr acquisition simulation from an acquired .zarr store # The script copies and writes additional metadata to .zattrs inserting two keys @@ -17,7 +18,7 @@ # Refer to steps at the end of the file on steps to run this file #%% ############################################# -def convertData(tif_path, latest_out_path, prefix="", data_type_str="ometiff"): +def convert_data(tif_path, latest_out_path, prefix="", data_type_str="ometiff"): converter = TIFFConverter( os.path.join(tif_path , prefix), latest_out_path, @@ -26,13 +27,13 @@ def convertData(tif_path, latest_out_path, prefix="", data_type_str="ometiff"): ) converter.run() -def runConvert(ome_tif_path): +def run_convert(ome_tif_path): out_path = os.path.join(Path(ome_tif_path).parent.absolute(), ("raw_" + Path(ome_tif_path).name + ".zarr")) - convertData(ome_tif_path, out_path) + convert_data(ome_tif_path, out_path) #%% ############################################# -def runAcq(input_path="", waitBetweenT=30): +def run_acq(input_path="", waitBetweenT=30): output_store_path = os.path.join(Path(input_path).parent.absolute(), ("acq_sim_" + Path(input_path).name)) @@ -106,13 +107,47 @@ def runAcq(input_path="", waitBetweenT=30): my_dict = output_dataset.zattrs["CurrentDimensions"] sorted_dict_acq = {k: my_dict[k] for k in sorted(my_dict, key=lambda x: required_order.index(x))} print("Writer thread - Acquisition Dim:", sorted_dict_acq) + + + # reconThread = threading.Thread(target=doReconstruct, args=(output_store_path, t)) + # reconThread.start() + time.sleep(waitBetweenT) # sleep after every t output_dataset.close +def do_reconstruct(input_path, time_point): + + config_path = os.path.join(Path(input_path).parent.absolute(), "Bire-"+str(time_point)+".yml") + output_path = os.path.join(Path(input_path).parent.absolute(), "Recon_"+Path(input_path).name) + mainfp = str(jobs_mgmt.FILE_PATH) + + print("Processing {input} time_point={tp}".format(input=input_path, tp=time_point)) + + try: + proc = subprocess.run( + [ + "python", + mainfp, + "reconstruct", + "-i", + input_path, + "-c", + config_path, + "-o", + output_path, + "-rx", + str(20) + ] + ) + if proc.returncode != 0: + raise Exception("An error occurred in processing ! Check terminal output.") + except Exception as exc: + print(exc.args) + #%% ############################################# -def runAcquire(input_path, waitBetweenT): - runThread1Acq = threading.Thread(target=runAcq, args=(input_path, waitBetweenT)) +def run_acquire(input_path, waitBetweenT): + runThread1Acq = threading.Thread(target=run_acq, args=(input_path, waitBetweenT)) runThread1Acq.start() #%% ############################################# @@ -128,8 +163,8 @@ def runAcquire(input_path, waitBetweenT): # run the test to simulate Acquiring a recOrder .zarr store input_path = "/ome-zarr_data/recOrderAcq/test/raw_snap_6D_ometiff_1.zarr" -waitBetweenT = 90 -runAcquire(input_path, waitBetweenT) +waitBetweenT = 60 +run_acquire(input_path, waitBetweenT) From b0165963deeb6638025586dfb68769443f00599e Mon Sep 17 00:00:00 2001 From: Amitabh Verma Date: Wed, 22 Jan 2025 11:28:39 -0500 Subject: [PATCH 38/38] with rx default as 1, catch and report OOM errors --- recOrder/plugin/tab_recon.py | 54 ++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/recOrder/plugin/tab_recon.py b/recOrder/plugin/tab_recon.py index fafec751..6a14b526 100644 --- a/recOrder/plugin/tab_recon.py +++ b/recOrder/plugin/tab_recon.py @@ -81,6 +81,7 @@ JOB_COMPLETION_STR = "Job completed successfully" JOB_RUNNING_STR = "Starting with JobEnvironment" JOB_TRIGGERED_EXC = "Submitted job triggered an exception" +JOB_OOM_EVENT = "oom_kill event" _validate_alert = "⚠" _validate_ok = "✔️" @@ -3310,6 +3311,21 @@ def table_update_and_cleaup_thread( logs_folder_path, extension="err", ) + if JOB_OOM_EVENT in jobERR: + params["status"] = STATUS_errored_job + _infoBox.setText( + jobERR + + "\n\n" + + jobTXT + ) + self.client_release( + expIdx, + jobIdx, + client_socket, + params, + reason=0, + ) + break _infoBox.setText( jobIdx + "\n" @@ -3323,7 +3339,7 @@ def table_update_and_cleaup_thread( jobIdx, client_socket, params, - reason=6, + reason=0, ) break elif params["status"] == STATUS_finished_job: @@ -3336,7 +3352,7 @@ def table_update_and_cleaup_thread( jobIdx, client_socket, params, - reason=7, + reason=6, ) break else: @@ -3368,7 +3384,7 @@ def table_update_and_cleaup_thread( jobIdx, client_socket, params, - reason=8, + reason=0, ) break elif JOB_RUNNING_STR in jobTXT: @@ -3376,7 +3392,27 @@ def table_update_and_cleaup_thread( _infoBox.setText(jobTXT) _tUpdateCount += 1 if _tUpdateCount > 60: - if _lastUpdate_jobTXT != jobTXT: + jobERR = self.JobsMgmt.check_for_jobID_File( + jobIdx, + logs_folder_path, + extension="err", + ) + if JOB_OOM_EVENT in jobERR: + params["status"] = STATUS_errored_job + _infoBox.setText( + jobERR + + "\n\n" + + jobTXT + ) + self.client_release( + expIdx, + jobIdx, + client_socket, + params, + reason=0, + ) + break + elif _lastUpdate_jobTXT != jobTXT: # if there is an update reset counter _tUpdateCount = 0 _lastUpdate_jobTXT = jobTXT @@ -3391,7 +3427,7 @@ def table_update_and_cleaup_thread( jobIdx, client_socket, params, - reason=9, + reason=0, ) break else: @@ -3410,14 +3446,14 @@ def table_update_and_cleaup_thread( jobIdx, client_socket, params, - reason=10, + reason=0, ) break except Exception as exc: print(exc.args) else: self.client_release( - expIdx, jobIdx, client_socket, params, reason=11 + expIdx, jobIdx, client_socket, params, reason=0 ) break else: @@ -3460,7 +3496,9 @@ def client_release(self, expIdx, jobIdx, client_socket, params, reason=0): } json_str = json.dumps(json_obj) + "\n" client_socket.send(json_str.encode()) - ROW_POP_QUEUE.append(expIdx) + + if reason != 0: # remove processing entry when exiting without error + ROW_POP_QUEUE.append(expIdx) # print("FINISHED") if self.pool is not None: