From c0392652473b2809fcf32b222be618448bc1861d Mon Sep 17 00:00:00 2001 From: Francisco Hernandez Vivanco <49014169+fhernandezvivanco@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:51:02 +1000 Subject: [PATCH] update zmq start message with data loaded from the master file (#8) * update zmq start message with data loaded from the master file * bump version * set saturation_value to 33000 if saturation_value key does not exist * add endpoint to set and the delay between frames * update default delay_between_frames value * parse correct value of detector_translation * change dtype when master file is loaded * use tolist to safely convert numpy array to lists * read detectot config from hdf5 file * fix typo * fix detectot config error * adds more detector config params from hdf5 file --- README.md | 7 +- ansto_simplon_api/parse_master_file.py | 43 ++++++++-- .../routes/ansto_endpoints/load_hdf5_files.py | 14 ++++ ansto_simplon_api/routes/detector/config.py | 24 +++--- ansto_simplon_api/schemas/configuration.py | 9 ++- ansto_simplon_api/simulate_zmq_stream.py | 80 +++++++++++++++++-- pyproject.toml | 2 +- 7 files changed, 142 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 24b2f0a..8178d40 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,10 @@ Currently generates a [Stream V2] release compatible ZMQ stream. To run the simulated Simplon API, you need to specify the path of an HDF5 master file using the `HDF5_MASTER_FILE` environment variable. You can also configure other parameters using the following environment variables: - - `DELAY_BETWEEN_FRAMES`: Specifies the delay between frames in seconds (default: 0.1 s). - - `NUMBER_OF_DATA_FILES`: Sets the number of data files from the master file loaded into memory (default: 1). Note that the datafiles are stored in memory, so they should not be too large. - - `NUMBER_OF_FRAMES_PER_TRIGGER`: Controls the number of frames per trigger. By default, it's set to 30, but you can modify it using the `/detector/api/1.8.0/config/nimages` endpoint. + - `DELAY_BETWEEN_FRAMES`: Specifies the delay between frames in seconds (default: 0.01 s). This number can be modified via the `/ansto_endpoints/delay_between_frames` endpoint. + - `NUMBER_OF_DATA_FILES`: Sets the number of data files from the master file loaded into memory (default: 1). The number of datafiles can be additionally modified when loading a new master file using the + `/ansto_endpoints/hdf5_master_file` endpoint. + - The number of frames per trigger is set automatically to the number of frames in the master file. This can be modified by using the `/detector/api/1.8.0/config/nimages` endpoint. ## Running the simulated SIMPLON API diff --git a/ansto_simplon_api/parse_master_file.py b/ansto_simplon_api/parse_master_file.py index 21e185e..12c3e2b 100644 --- a/ansto_simplon_api/parse_master_file.py +++ b/ansto_simplon_api/parse_master_file.py @@ -1,6 +1,14 @@ +import logging + import h5py import numpy as np +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s: %(message)s", + datefmt="%d-%m-%Y %H:%M:%S", +) + class Parse: """ @@ -91,6 +99,18 @@ def header(self) -> dict: message structure. """ + try: + saturation_value = int( + np.array( + self.hf["/entry/instrument/detector/saturation_value"] + ).tolist() + ) + except KeyError: + logging.warning( + "/entry/instrument/detector/saturation_value was not found in the master file. " + "Setting saturation value to 33000" + ) + saturation_value = 33000 start_message = { "type": "start", "arm_date": self.parse("data_collection_date"), @@ -104,11 +124,9 @@ def header(self) -> dict: "countrate_correction_lookup_table": None, "detector_description": self.parse("description"), "detector_serial_number": self.parse("detector_number"), - "detector_translation": [ - 0.0, - 0.0, - self.parse("detector_distance"), - ], + "detector_translation": np.array( + self.hf["/entry/instrument/detector/geometry/translation/distances"] + ).tolist(), "flatfield": None, "flatfield_enabled": bool(self.parse("flatfield_correction_applied")), "frame_time": self.parse("frame_time"), @@ -127,12 +145,21 @@ def header(self) -> dict: "image_size_y": self.parse("y_pixels_in_detector"), "incident_energy": self.parse("photon_energy"), "incident_wavelength": self.parse("incident_wavelength"), - "number_of_images": None, # self.parse("nimages"), + "number_of_images": int( + np.array( + self.hf["/entry/instrument/detector/detectorSpecific/ntrigger"] + ).tolist() + ) + * int( + np.array( + self.hf["/entry/instrument/detector/detectorSpecific/nimages"] + ).tolist() + ), "pixel_mask": None, "pixel_mask_enabled": bool(self.parse("pixel_mask_applied")), "pixel_size_x": self.parse("x_pixel_size"), "pixel_size_y": self.parse("y_pixel_size"), - "saturation_value": self.parse("saturation_value"), + "saturation_value": saturation_value, "sensor_material": self.parse("sensor_material"), "sensor_thickness": self.parse("sensor_thickness"), "series_id": None, # int @@ -142,7 +169,7 @@ def header(self) -> dict: "threshold_2": self.parse("threshold_energy") * 3, }, "user_data": {"pi": float(np.pi)}, - "virtual_pixel_correction_applied": bool( + "virtual_pixel_interpolation_enabled": bool( self.parse("virtual_pixel_correction_applied") ), } diff --git a/ansto_simplon_api/routes/ansto_endpoints/load_hdf5_files.py b/ansto_simplon_api/routes/ansto_endpoints/load_hdf5_files.py index 749f27f..b1c7ad6 100644 --- a/ansto_simplon_api/routes/ansto_endpoints/load_hdf5_files.py +++ b/ansto_simplon_api/routes/ansto_endpoints/load_hdf5_files.py @@ -3,6 +3,7 @@ from starlette import status from ...schemas.ansto_endpoints import LoadHDF5File +from ...schemas.configuration import SimplonRequestFloat from ...simulate_zmq_stream import zmq_stream router = APIRouter(prefix="/ansto_endpoints", tags=["ANSTO Endpoints"]) @@ -38,3 +39,16 @@ async def get_master_file() -> LoadHDF5File: number_of_datafiles=zmq_stream.number_of_data_files, compression=zmq_stream.compression, ) + + +@router.get("/delay_between_frames") +async def get_delay_between_frames_in_seconds() -> SimplonRequestFloat: + return SimplonRequestFloat(value=zmq_stream.delay_between_frames) + + +@router.put("/delay_between_frames") +async def set_delay_between_frames_in_seconds( + delay: SimplonRequestFloat, +) -> SimplonRequestFloat: + zmq_stream.delay_between_frames = delay.value + return SimplonRequestFloat(value=zmq_stream.delay_between_frames) diff --git a/ansto_simplon_api/routes/detector/config.py b/ansto_simplon_api/routes/detector/config.py index e53f1b2..8acd7f5 100644 --- a/ansto_simplon_api/routes/detector/config.py +++ b/ansto_simplon_api/routes/detector/config.py @@ -1,7 +1,6 @@ from fastapi import APIRouter from ...schemas.configuration import ( - DetectorConfiguration, SimplonRequestAny, SimplonRequestBool, SimplonRequestDict, @@ -13,7 +12,6 @@ router = APIRouter(prefix="/detector/api/1.8.0/config", tags=["Detector Configuration"]) -detector_configuration = DetectorConfiguration() ### Detector subsystem config # @router.put("/auto_summation") @@ -48,20 +46,20 @@ async def put_beam_center_y(input: SimplonRequestFloat): @router.put("/bit_depth_image") async def put_bit_depth_image(input: SimplonRequestInt): - detector_configuration.detector_bit_depth_image = input.value + zmq_stream.detector_config.detector_bit_depth_image = input.value # the bit depth image is not the dtype - return {"value": detector_configuration.detector_bit_depth_image} + return {"value": zmq_stream.detector_config.detector_bit_depth_image} @router.get("/bit_depth_image") async def get_bit_depth_image(): - return {"value": detector_configuration.detector_bit_depth_image} + return {"value": zmq_stream.detector_config.detector_bit_depth_image} # @router.put("/bit_depth_readout") @router.get("/bit_depth_readout") async def get_bit_depth_readout(): - return {"value": 16} + return {"value": zmq_stream.detector_config.detector_bit_depth_readout} # chi_increment @@ -106,7 +104,7 @@ async def put_countrate_correction_applied(input: SimplonRequestBool): # @router.put("/countrate_correction_count_cutoff") @router.get("/countrate_correction_count_cutoff") async def get_countrate_correction_count_cutoff(): - return {"value": 133343} + return {"value": zmq_stream.detector_config.detector_countrate_correction_cutoff} # data_collection_date @@ -150,19 +148,19 @@ async def get_detector_number(): @router.get("/detector_readout_time") async def get_detector_readout_time(): - return {"value": detector_configuration.detector_readout_time} + return {"value": zmq_stream.detector_config.detector_readout_time} @router.put("/detector_readout_time") async def put_detector_readout_time(input: SimplonRequestFloat): - detector_configuration.detector_readout_time = input.value - return {"value": detector_configuration.detector_readout_time} + zmq_stream.detector_config.detector_readout_time = input.value + return {"value": zmq_stream.detector_config.detector_readout_time} # @router.put("/eiger_fw_version") @router.get("/eiger_fw_version") async def get_eiger_fw_version(): - return {"value": "release-2020.2.5"} + return {"value": zmq_stream.detector_config.eiger_fw_version} # element @@ -264,7 +262,7 @@ async def get_sensor_thickness(): # @router.put("/software_version") @router.get("/software_version") async def get_software_version(): - return {"value": "1.8.0"} + return {"value": zmq_stream.detector_config.software_version} # threshold_energy @@ -289,7 +287,7 @@ async def put_threshold_energy(input: SimplonRequestDict): # @router.put("/trigger_mode") @router.get("/trigger_mode") async def get_trigger_mode(): - return {"value": "exts"} + return {"value": zmq_stream.detector_config.detector_trigger_mode} # trigger_start_delay diff --git a/ansto_simplon_api/schemas/configuration.py b/ansto_simplon_api/schemas/configuration.py index 34a4538..6ee5ddc 100644 --- a/ansto_simplon_api/schemas/configuration.py +++ b/ansto_simplon_api/schemas/configuration.py @@ -47,7 +47,7 @@ class ZMQStartMessage(BaseModel): countrate_correction_lookup_table: list | None = [0] detector_description: str = "Dectris EIGER2 Si 16M" detector_serial_number: str = "E-32-0130" - detector_translation: tuple[float, float, float] = [0, 0, -0.298] + detector_translation: tuple[float, float, float] | list = [0, 0, -0.298] flatfield: list | None = [] flatfield_enabled: bool = True frame_time: float = 0.0110 @@ -65,8 +65,8 @@ class ZMQStartMessage(BaseModel): saturation_value: int | None = 33000 # TODO: check where this value comes from sensor_material: str = "Si" sensor_thickness: float = 4.5e-04 - series_id: int = 0 - series_unique_id: str = 0 + series_id: int | None = 0 + series_unique_id: str | None = 0 threshold_energy: dict = {"threshold_1": 6350} user_data: dict | None = {} virtual_pixel_interpolation_enabled: bool = True @@ -78,9 +78,10 @@ class DetectorConfiguration(BaseModel): detector_readout_time: float = 0.0000001 detector_bit_depth_image: int = 32 detector_bit_depth_readout: int = 16 - detector_readout_time: float = 1e-07 detector_compression: str = "bslz4" detector_countrate_correction_cutoff: int = 126634 detector_ntrigger: int = 1 detector_number_of_excluded_pixels: int = 1251206 detector_trigger_mode: str = "exts" + software_version: str = "E-32-0130" + eiger_fw_version: str = "release-2022.1.2rc2" diff --git a/ansto_simplon_api/simulate_zmq_stream.py b/ansto_simplon_api/simulate_zmq_stream.py index b8c628f..abbba05 100644 --- a/ansto_simplon_api/simulate_zmq_stream.py +++ b/ansto_simplon_api/simulate_zmq_stream.py @@ -16,7 +16,7 @@ from tqdm import trange from .parse_master_file import Parse -from .schemas.configuration import ZMQStartMessage +from .schemas.configuration import DetectorConfiguration, ZMQStartMessage logging.basicConfig( level=logging.INFO, @@ -40,7 +40,6 @@ def __init__( hdf5_file_path: str, delay_between_frames: float = 0.1, number_of_data_files: int = 1, - number_of_frames_per_trigger: int = 200, ) -> None: """ Parameters @@ -53,8 +52,6 @@ def __init__( Time delay between images sent via the ZeroMQ stream [seconds] number_of_data_files : int, optional Number of data files loaded in memory - number_of_frames_per_trigger : int, optional - Number of frames per trigger Returns ------- @@ -65,7 +62,7 @@ def __init__( self.compression = "bslz4" self.delay_between_frames = delay_between_frames self.number_of_data_files = number_of_data_files - self.number_of_frames_per_trigger = number_of_frames_per_trigger + self.number_of_frames_per_trigger = None self.context = zmq.Context() self.socket = self.context.socket(zmq.PUSH) @@ -82,6 +79,8 @@ def __init__( self.series_unique_id = None self.frames = None self.hdf5_file_path = hdf5_file_path + self.detector_config = DetectorConfiguration() + self.create_list_of_compressed_frames( self.hdf5_file_path, self.compression, self.number_of_data_files ) @@ -92,6 +91,68 @@ def __init__( logging.info(f"Delay between frames (s): {self.delay_between_frames}") logging.info(f"Number of data files: {self.number_of_data_files}") + def _update_zmq_start_message(self) -> None: + """ + Updates the ZMQ start message with values derived from the master file + loaded into the Simplon API. + + Returns + ------- + None + """ + for key, val in self.start_message.items(): + setattr(zmq_start_message, key, val) + + def _update_detector_configuration(self, hf: h5py.File) -> None: + """ + Updates the detector configuration by reading the detector + config from a hdf5 file + + Parameters + ---------- + hf : h5py.File + A hdf5 file + + Returns + ------- + None + """ + try: + self.detector_config.detector_readout_time = float( + hf["/entry/instrument/detector/detector_readout_time"][()] + ) + self.detector_config.detector_bit_depth_image = int( + hf["/entry/instrument/detector/bit_depth_image"][()] + ) + self.detector_config.detector_bit_depth_readout = int( + hf["/entry/instrument/detector/bit_depth_readout"][()] + ) + self.detector_config.detector_compression = str( + hf["/entry/instrument/detector/detectorSpecific/compression"][ + () + ].decode() + ) + self.detector_config.detector_countrate_correction_cutoff = int( + hf[ + "/entry/instrument/detector/detectorSpecific/countrate_correction_count_cutoff" # noqa + ][()] + ) + self.detector_config.software_version = str( + hf["/entry/instrument/detector/detectorSpecific/software_version"][ + () + ].decode() + ) + self.detector_config.eiger_fw_version = str( + hf["/entry/instrument/detector/detectorSpecific/eiger_fw_version"][ + () + ].decode() + ) + except KeyError: + logging.warning( + "Detector configuration could not be loaded. Using detector " + "configuration defaults" + ) + def create_list_of_compressed_frames( self, hdf5_file_path: str, @@ -143,6 +204,10 @@ def create_list_of_compressed_frames( self.start_message, self.image_message, self.end_message = Parse( hdf5_file ).header() + self._update_zmq_start_message() + self._update_detector_configuration(hdf5_file) + + self.number_of_frames_per_trigger = zmq_start_message.number_of_images number_of_frames_per_data_file = [ datafile.shape[0] for datafile in datafile_list @@ -153,6 +218,7 @@ def create_list_of_compressed_frames( zmq_start_message.image_size_y = array_shape[0] dtype = datafile_list[0].dtype + zmq_start_message.image_dtype = str(dtype) frame_list = [] @@ -363,14 +429,12 @@ def start_stream(self) -> None: "HDF5_MASTER_FILE environment variable" ) -DELAY_BETWEEN_FRAMES = float(environ.get("DELAY_BETWEEN_FRAMES", "0.1")) +DELAY_BETWEEN_FRAMES = float(environ.get("DELAY_BETWEEN_FRAMES", "0.01")) NUMBER_OF_DATA_FILES = int(environ.get("NUMBER_OF_DATA_FILES", "1")) -NUMBER_OF_FRAMES_PER_TRIGGER = int(environ.get("NUMBER_OF_FRAMES_PER_TRIGGER", "1")) zmq_stream = ZmqStream( address=ZMQ_ADDRESS, hdf5_file_path=HDF5_MASTER_FILE, delay_between_frames=DELAY_BETWEEN_FRAMES, number_of_data_files=NUMBER_OF_DATA_FILES, - number_of_frames_per_trigger=NUMBER_OF_FRAMES_PER_TRIGGER, ) diff --git a/pyproject.toml b/pyproject.toml index 4d482a3..a984cb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ansto-simplon-api" -version = "0.2.4" +version = "0.2.5" description = "Simulated simplon api" authors = ["Francisco Hernandez Vivanco ", "Daniel Eriksson "]