update zmq start message with data loaded from the master file (#8)

* update zmq start message with data loaded from the master file

* bump version

* set saturation_value to 33000 if saturation_value key does not  exist

* add endpoint to set and the delay between frames

* update default delay_between_frames value

* parse correct value of detector_translation

* change dtype when master file is loaded

* use tolist to safely convert numpy array to lists

* read detectot config from hdf5 file

* fix typo

* fix detectot config error

* adds more detector config params from hdf5 file

README.md

@@ -10,9 +10,10 @@ Currently generates a [Stream V2] release compatible ZMQ stream.
 
    To run the simulated Simplon API, you need to specify the path of an HDF5 master file using the `HDF5_MASTER_FILE` environment variable. You can also configure other parameters using the following environment variables:
 
-   - `DELAY_BETWEEN_FRAMES`: Specifies the delay between frames in seconds (default: 0.1 s).
-   - `NUMBER_OF_DATA_FILES`: Sets the number of data files from the master file loaded into memory (default: 1). Note that the datafiles are stored in memory, so they should not be too large.
-   - `NUMBER_OF_FRAMES_PER_TRIGGER`: Controls the number of frames per trigger. By default, it's set to 30, but you can modify it using the `/detector/api/1.8.0/config/nimages` endpoint.
+   - `DELAY_BETWEEN_FRAMES`: Specifies the delay between frames in seconds (default: 0.01 s). This number can be modified via the `/ansto_endpoints/delay_between_frames` endpoint.
+   - `NUMBER_OF_DATA_FILES`: Sets the number of data files from the master file loaded into memory (default: 1). The number of datafiles can be additionally modified when loading a new master file using the
+   `/ansto_endpoints/hdf5_master_file` endpoint.
+   - The number of frames per trigger is set automatically to the number of frames in the master file. This can be modified by using the `/detector/api/1.8.0/config/nimages` endpoint.
 
 ## Running the simulated SIMPLON API
 

ansto_simplon_api/parse_master_file.py

@@ -1,6 +1,14 @@
+import logging
+
 import h5py
 import numpy as np
 
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s: %(message)s",
+    datefmt="%d-%m-%Y %H:%M:%S",
+)
+
 
 class Parse:
     """
@@ -91,6 +99,18 @@ def header(self) -> dict:
             message structure.
 
         """
+        try:
+            saturation_value = int(
+                np.array(
+                    self.hf["/entry/instrument/detector/saturation_value"]
+                ).tolist()
+            )
+        except KeyError:
+            logging.warning(
+                "/entry/instrument/detector/saturation_value was not found in the master file. "
+                "Setting saturation value to 33000"
+            )
+            saturation_value = 33000
         start_message = {
             "type": "start",
             "arm_date": self.parse("data_collection_date"),
@@ -104,11 +124,9 @@ def header(self) -> dict:
             "countrate_correction_lookup_table": None,
             "detector_description": self.parse("description"),
             "detector_serial_number": self.parse("detector_number"),
-            "detector_translation": [
-                0.0,
-                0.0,
-                self.parse("detector_distance"),
-            ],
+            "detector_translation": np.array(
+                self.hf["/entry/instrument/detector/geometry/translation/distances"]
+            ).tolist(),
             "flatfield": None,
             "flatfield_enabled": bool(self.parse("flatfield_correction_applied")),
             "frame_time": self.parse("frame_time"),
@@ -127,12 +145,21 @@ def header(self) -> dict:
             "image_size_y": self.parse("y_pixels_in_detector"),
             "incident_energy": self.parse("photon_energy"),
             "incident_wavelength": self.parse("incident_wavelength"),
-            "number_of_images": None,  # self.parse("nimages"),
+            "number_of_images": int(
+                np.array(
+                    self.hf["/entry/instrument/detector/detectorSpecific/ntrigger"]
+                ).tolist()
+            )
+            * int(
+                np.array(
+                    self.hf["/entry/instrument/detector/detectorSpecific/nimages"]
+                ).tolist()
+            ),
             "pixel_mask": None,
             "pixel_mask_enabled": bool(self.parse("pixel_mask_applied")),
             "pixel_size_x": self.parse("x_pixel_size"),
             "pixel_size_y": self.parse("y_pixel_size"),
-            "saturation_value": self.parse("saturation_value"),
+            "saturation_value": saturation_value,
             "sensor_material": self.parse("sensor_material"),
             "sensor_thickness": self.parse("sensor_thickness"),
             "series_id": None,  # int
@@ -142,7 +169,7 @@ def header(self) -> dict:
                 "threshold_2": self.parse("threshold_energy") * 3,
             },
             "user_data": {"pi": float(np.pi)},
-            "virtual_pixel_correction_applied": bool(
+            "virtual_pixel_interpolation_enabled": bool(
                 self.parse("virtual_pixel_correction_applied")
             ),
         }

ansto_simplon_api/routes/ansto_endpoints/load_hdf5_files.py

@@ -3,6 +3,7 @@
 from starlette import status
 
 from ...schemas.ansto_endpoints import LoadHDF5File
+from ...schemas.configuration import SimplonRequestFloat
 from ...simulate_zmq_stream import zmq_stream
 
 router = APIRouter(prefix="/ansto_endpoints", tags=["ANSTO Endpoints"])
@@ -38,3 +39,16 @@ async def get_master_file() -> LoadHDF5File:
         number_of_datafiles=zmq_stream.number_of_data_files,
         compression=zmq_stream.compression,
     )
+
+
+@router.get("/delay_between_frames")
+async def get_delay_between_frames_in_seconds() -> SimplonRequestFloat:
+    return SimplonRequestFloat(value=zmq_stream.delay_between_frames)
+
+
+@router.put("/delay_between_frames")
+async def set_delay_between_frames_in_seconds(
+    delay: SimplonRequestFloat,
+) -> SimplonRequestFloat:
+    zmq_stream.delay_between_frames = delay.value
+    return SimplonRequestFloat(value=zmq_stream.delay_between_frames)

ansto_simplon_api/routes/detector/config.py

@@ -1,7 +1,6 @@
 from fastapi import APIRouter
 
 from ...schemas.configuration import (
-    DetectorConfiguration,
     SimplonRequestAny,
     SimplonRequestBool,
     SimplonRequestDict,
@@ -13,7 +12,6 @@
 
 router = APIRouter(prefix="/detector/api/1.8.0/config", tags=["Detector Configuration"])
 
-detector_configuration = DetectorConfiguration()
 
 ### Detector subsystem config
 # @router.put("/auto_summation")
@@ -48,20 +46,20 @@ async def put_beam_center_y(input: SimplonRequestFloat):
 
 @router.put("/bit_depth_image")
 async def put_bit_depth_image(input: SimplonRequestInt):
-    detector_configuration.detector_bit_depth_image = input.value
+    zmq_stream.detector_config.detector_bit_depth_image = input.value
     # the bit depth image is not the dtype
-    return {"value": detector_configuration.detector_bit_depth_image}
+    return {"value": zmq_stream.detector_config.detector_bit_depth_image}
 
 
 @router.get("/bit_depth_image")
 async def get_bit_depth_image():
-    return {"value": detector_configuration.detector_bit_depth_image}
+    return {"value": zmq_stream.detector_config.detector_bit_depth_image}
 
 
 # @router.put("/bit_depth_readout")
 @router.get("/bit_depth_readout")
 async def get_bit_depth_readout():
-    return {"value": 16}
+    return {"value": zmq_stream.detector_config.detector_bit_depth_readout}
 
 
 # chi_increment
@@ -106,7 +104,7 @@ async def put_countrate_correction_applied(input: SimplonRequestBool):
 # @router.put("/countrate_correction_count_cutoff")
 @router.get("/countrate_correction_count_cutoff")
 async def get_countrate_correction_count_cutoff():
-    return {"value": 133343}
+    return {"value": zmq_stream.detector_config.detector_countrate_correction_cutoff}
 
 
 # data_collection_date
@@ -150,19 +148,19 @@ async def get_detector_number():
 
 @router.get("/detector_readout_time")
 async def get_detector_readout_time():
-    return {"value": detector_configuration.detector_readout_time}
+    return {"value": zmq_stream.detector_config.detector_readout_time}
 
 
 @router.put("/detector_readout_time")
 async def put_detector_readout_time(input: SimplonRequestFloat):
-    detector_configuration.detector_readout_time = input.value
-    return {"value": detector_configuration.detector_readout_time}
+    zmq_stream.detector_config.detector_readout_time = input.value
+    return {"value": zmq_stream.detector_config.detector_readout_time}
 
 
 # @router.put("/eiger_fw_version")
 @router.get("/eiger_fw_version")
 async def get_eiger_fw_version():
-    return {"value": "release-2020.2.5"}
+    return {"value": zmq_stream.detector_config.eiger_fw_version}
 
 
 # element
@@ -264,7 +262,7 @@ async def get_sensor_thickness():
 # @router.put("/software_version")
 @router.get("/software_version")
 async def get_software_version():
-    return {"value": "1.8.0"}
+    return {"value": zmq_stream.detector_config.software_version}
 
 
 # threshold_energy
@@ -289,7 +287,7 @@ async def put_threshold_energy(input: SimplonRequestDict):
 # @router.put("/trigger_mode")
 @router.get("/trigger_mode")
 async def get_trigger_mode():
-    return {"value": "exts"}
+    return {"value": zmq_stream.detector_config.detector_trigger_mode}
 
 
 # trigger_start_delay

ansto_simplon_api/schemas/configuration.py

@@ -47,7 +47,7 @@ class ZMQStartMessage(BaseModel):
     countrate_correction_lookup_table: list | None = [0]
     detector_description: str = "Dectris EIGER2 Si 16M"
     detector_serial_number: str = "E-32-0130"
-    detector_translation: tuple[float, float, float] = [0, 0, -0.298]
+    detector_translation: tuple[float, float, float] | list = [0, 0, -0.298]
     flatfield: list | None = []
     flatfield_enabled: bool = True
     frame_time: float = 0.0110
@@ -65,8 +65,8 @@ class ZMQStartMessage(BaseModel):
     saturation_value: int | None = 33000  # TODO: check where this value comes from
     sensor_material: str = "Si"
     sensor_thickness: float = 4.5e-04
-    series_id: int = 0
-    series_unique_id: str = 0
+    series_id: int | None = 0
+    series_unique_id: str | None = 0
     threshold_energy: dict = {"threshold_1": 6350}
     user_data: dict | None = {}
     virtual_pixel_interpolation_enabled: bool = True
@@ -78,9 +78,10 @@ class DetectorConfiguration(BaseModel):
     detector_readout_time: float = 0.0000001
     detector_bit_depth_image: int = 32
     detector_bit_depth_readout: int = 16
-    detector_readout_time: float = 1e-07
     detector_compression: str = "bslz4"
     detector_countrate_correction_cutoff: int = 126634
     detector_ntrigger: int = 1
     detector_number_of_excluded_pixels: int = 1251206
     detector_trigger_mode: str = "exts"
+    software_version: str = "E-32-0130"
+    eiger_fw_version: str = "release-2022.1.2rc2"

ansto_simplon_api/simulate_zmq_stream.py

@@ -16,7 +16,7 @@
 from tqdm import trange
 
 from .parse_master_file import Parse
-from .schemas.configuration import ZMQStartMessage
+from .schemas.configuration import DetectorConfiguration, ZMQStartMessage
 
 logging.basicConfig(
     level=logging.INFO,
@@ -40,7 +40,6 @@ def __init__(
         hdf5_file_path: str,
         delay_between_frames: float = 0.1,
         number_of_data_files: int = 1,
-        number_of_frames_per_trigger: int = 200,
     ) -> None:
         """
         Parameters
@@ -53,8 +52,6 @@ def __init__(
             Time delay between images sent via the ZeroMQ stream [seconds]
         number_of_data_files : int, optional
             Number of data files loaded in memory
-        number_of_frames_per_trigger : int, optional
-            Number of frames per trigger
 
         Returns
         -------
@@ -65,7 +62,7 @@ def __init__(
         self.compression = "bslz4"
         self.delay_between_frames = delay_between_frames
         self.number_of_data_files = number_of_data_files
-        self.number_of_frames_per_trigger = number_of_frames_per_trigger
+        self.number_of_frames_per_trigger = None
 
         self.context = zmq.Context()
         self.socket = self.context.socket(zmq.PUSH)
@@ -82,6 +79,8 @@ def __init__(
         self.series_unique_id = None
         self.frames = None
         self.hdf5_file_path = hdf5_file_path
+        self.detector_config = DetectorConfiguration()
+
         self.create_list_of_compressed_frames(
             self.hdf5_file_path, self.compression, self.number_of_data_files
         )
@@ -92,6 +91,68 @@ def __init__(
         logging.info(f"Delay between frames (s): {self.delay_between_frames}")
         logging.info(f"Number of data files: {self.number_of_data_files}")
 
+    def _update_zmq_start_message(self) -> None:
+        """
+        Updates the ZMQ start message with values derived from the master file
+        loaded into the Simplon API.
+
+        Returns
+        -------
+        None
+        """
+        for key, val in self.start_message.items():
+            setattr(zmq_start_message, key, val)
+
+    def _update_detector_configuration(self, hf: h5py.File) -> None:
+        """
+        Updates the detector configuration by reading the detector
+        config from a hdf5 file
+
+        Parameters
+        ----------
+        hf : h5py.File
+            A hdf5 file
+
+        Returns
+        -------
+        None
+        """
+        try:
+            self.detector_config.detector_readout_time = float(
+                hf["/entry/instrument/detector/detector_readout_time"][()]
+            )
+            self.detector_config.detector_bit_depth_image = int(
+                hf["/entry/instrument/detector/bit_depth_image"][()]
+            )
+            self.detector_config.detector_bit_depth_readout = int(
+                hf["/entry/instrument/detector/bit_depth_readout"][()]
+            )
+            self.detector_config.detector_compression = str(
+                hf["/entry/instrument/detector/detectorSpecific/compression"][
+                    ()
+                ].decode()
+            )
+            self.detector_config.detector_countrate_correction_cutoff = int(
+                hf[
+                    "/entry/instrument/detector/detectorSpecific/countrate_correction_count_cutoff"  # noqa
+                ][()]
+            )
+            self.detector_config.software_version = str(
+                hf["/entry/instrument/detector/detectorSpecific/software_version"][
+                    ()
+                ].decode()
+            )
+            self.detector_config.eiger_fw_version = str(
+                hf["/entry/instrument/detector/detectorSpecific/eiger_fw_version"][
+                    ()
+                ].decode()
+            )
+        except KeyError:
+            logging.warning(
+                "Detector configuration could not be loaded. Using detector "
+                "configuration defaults"
+            )
+
     def create_list_of_compressed_frames(
         self,
         hdf5_file_path: str,
@@ -143,6 +204,10 @@ def create_list_of_compressed_frames(
             self.start_message, self.image_message, self.end_message = Parse(
                 hdf5_file
             ).header()
+            self._update_zmq_start_message()
+            self._update_detector_configuration(hdf5_file)
+
+        self.number_of_frames_per_trigger = zmq_start_message.number_of_images
 
         number_of_frames_per_data_file = [
             datafile.shape[0] for datafile in datafile_list
@@ -153,6 +218,7 @@ def create_list_of_compressed_frames(
         zmq_start_message.image_size_y = array_shape[0]
 
         dtype = datafile_list[0].dtype
+        zmq_start_message.image_dtype = str(dtype)
 
         frame_list = []
 
@@ -363,14 +429,12 @@ def start_stream(self) -> None:
         "HDF5_MASTER_FILE environment variable"
     )
 
-DELAY_BETWEEN_FRAMES = float(environ.get("DELAY_BETWEEN_FRAMES", "0.1"))
+DELAY_BETWEEN_FRAMES = float(environ.get("DELAY_BETWEEN_FRAMES", "0.01"))
 NUMBER_OF_DATA_FILES = int(environ.get("NUMBER_OF_DATA_FILES", "1"))
-NUMBER_OF_FRAMES_PER_TRIGGER = int(environ.get("NUMBER_OF_FRAMES_PER_TRIGGER", "1"))
 
 zmq_stream = ZmqStream(
     address=ZMQ_ADDRESS,
     hdf5_file_path=HDF5_MASTER_FILE,
     delay_between_frames=DELAY_BETWEEN_FRAMES,
     number_of_data_files=NUMBER_OF_DATA_FILES,
-    number_of_frames_per_trigger=NUMBER_OF_FRAMES_PER_TRIGGER,
 )