Merge pull request #116 from legend-exp/raw_bzip

Raw bzip

Author: ggmarshall

.pre-commit-config.yaml

@@ -78,7 +78,7 @@ repos:
     rev: "v2.3.0"
     hooks:
       - id: codespell
-        args: ["-L", "nd,unparseable,compiletime,livetime,fom,puls"]
+        args: ["-L", "nd,unparseable,compiletime,livetime,fom,puls,crate"]
 
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: "v0.10.0.1"

workflow/rules/raw.smk

@@ -48,6 +48,16 @@ rule build_raw_orca:
         "{params.ro_input} {output}"
 
 
+use rule build_raw_orca as build_raw_orca_bz2 with:
+    input:
+        get_pattern_tier_daq(config, extension="orca.bz2"),
+
+
+use rule build_raw_orca as build_raw_orca_gz with:
+    input:
+        get_pattern_tier_daq(config, extension="orca.gz"),
+
+
 rule build_raw_fcio:
     """
     This rule runs build_raw, it takes in a file.{daq_ext} and outputs a raw file

workflow/src/legenddataflow/FileKey.py

@@ -32,7 +32,12 @@ def regex_from_filepattern(filepattern):
             f.append(f"(?P={wildcard})")
         else:
             wildcards.append(wildcard)
-            f.append(f"(?P<{wildcard}>.+)")
+            if wildcard == "ext":
+                f.append(
+                    f"(?P<{wildcard}>.*)"
+                )  # this means ext will capture everything after 1st dot
+            else:
+                f.append(f"(?P<{wildcard}>" + r"[^\.\/]+)")
         last = match.end()
     f.append(re.escape(filepattern[last:]))
     f.append("$")

workflow/src/legenddataflow/create_pars_keylist.py

@@ -53,9 +53,14 @@ def generate_par_keylist(keys):
 
     @staticmethod
     def match_entries(entry1, entry2):
-        datatype2 = ProcessingFileKey.get_filekey_from_filename(entry2.file[0]).datatype
+        datatype2 = ProcessingFileKey.get_filekey_from_filename(
+            Path(entry2.file[0]).name
+        ).datatype
         for entry in entry1.file:
-            if ProcessingFileKey.get_filekey_from_filename(entry).datatype == datatype2:
+            if (
+                ProcessingFileKey.get_filekey_from_filename(Path(entry).name).datatype
+                == datatype2
+            ):
                 pass
             else:
                 entry2.file.append(entry)

workflow/src/legenddataflow/scripts/tier/raw_fcio.py

@@ -47,4 +47,10 @@ def build_tier_raw_fcio() -> None:
     if "muon_config" in channel_dict:
         raise NotImplementedError()
 
-    build_raw(args.input, out_spec=all_config, filekey=args.output, **settings)
+    build_raw(
+        args.input,
+        out_spec=all_config,
+        in_stream_type="Flashcam",
+        filekey=args.output,
+        **settings,
+    )

workflow/src/legenddataflow/scripts/tier/raw_orca.py

@@ -27,7 +27,7 @@ def build_tier_raw_orca() -> None:
 
     configs = TextDB(args.configs, lazy=True)
     config_dict = configs.on(args.timestamp, system=args.datatype)["snakemake_rules"][
-        "tier_raw"
+        "tier_raw_orca"
     ]
 
     build_log(config_dict, args.log)
@@ -63,11 +63,25 @@ def build_tier_raw_orca() -> None:
         spm_config[next(iter(spm_config))]["spms"]["key_list"] = sorted(spm_channels)
         Props.add_to(all_config, spm_config)
 
+    if "muon_config" in list(channel_dict):
+        muon_config = Props.read_from(channel_dict["muon_config"])
+        muon_channels = list(
+            chmap.channelmaps.on(args.timestamp)
+            .map("system", unique=False)["muon"]
+            .map("daq.rawid")
+        )
+        top_key = next(iter(muon_config))
+        muon_config[top_key][next(iter(muon_config[top_key]))]["key_list"] = sorted(
+            muon_channels
+        )
+        Props.add_to(all_config, muon_config)
+
     if "auxs_config" in list(channel_dict):
         aux_config = Props.read_from(channel_dict["auxs_config"])
         aux_channels = list(
             chmap.channelmaps.on(args.timestamp)
             .map("system", unique=False)["auxs"]
+            .map("daq.crate", unique=False)[1]
             .map("daq.rawid")
         )
         aux_channels += list(
@@ -86,17 +100,10 @@ def build_tier_raw_orca() -> None:
         )
         Props.add_to(all_config, aux_config)
 
-    if "muon_config" in list(channel_dict):
-        muon_config = Props.read_from(channel_dict["muon_config"])
-        muon_channels = list(
-            chmap.channelmaps.on(args.timestamp)
-            .map("system", unique=False)["muon"]
-            .map("daq.rawid")
-        )
-        top_key = next(iter(muon_config))
-        muon_config[top_key][next(iter(muon_config[top_key]))]["key_list"] = sorted(
-            muon_channels
-        )
-        Props.add_to(all_config, muon_config)
-
-    build_raw(args.input, out_spec=all_config, filekey=args.output, **settings)
+    build_raw(
+        args.input,
+        out_spec=all_config,
+        in_stream_type="ORCA",
+        filekey=args.output,
+        **settings,
+    )