add rules for decoding gzip and bzip orca files

Author: ggmarshall

workflow/rules/raw.smk

@@ -48,6 +48,62 @@ rule build_raw_orca:
         "{params.ro_input} {output}"
 
 
+rule build_raw_orca_bz2:
+    """
+    This rule runs build_raw, it takes in a file.{daq_ext} and outputs a raw file
+    """
+    input:
+        get_pattern_tier_daq(config, extension="orca.bz2"),
+    params:
+        timestamp="{timestamp}",
+        datatype="{datatype}",
+        ro_input=lambda _, input: ro(input),
+    output:
+        get_pattern_tier(config, "raw", check_in_cycle=check_in_cycle),
+    log:
+        get_pattern_log(config, "tier_raw", time),
+    group:
+        "tier-raw"
+    resources:
+        mem_swap=110,
+        runtime=300,
+    shell:
+        execenv_pyexe(config, "build-tier-raw-orca") + "--log {log} "
+        f"--configs {ro(configs)} "
+        f"--chan-maps {ro(chan_maps)} "
+        "--datatype {params.datatype} "
+        "--timestamp {params.timestamp} "
+        "{params.ro_input} {output}"
+
+
+rule build_raw_orca_gzip:
+    """
+    This rule runs build_raw, it takes in a file.{daq_ext} and outputs a raw file
+    """
+    input:
+        get_pattern_tier_daq(config, extension="orca.gz"),
+    params:
+        timestamp="{timestamp}",
+        datatype="{datatype}",
+        ro_input=lambda _, input: ro(input),
+    output:
+        get_pattern_tier(config, "raw", check_in_cycle=check_in_cycle),
+    log:
+        get_pattern_log(config, "tier_raw", time),
+    group:
+        "tier-raw"
+    resources:
+        mem_swap=110,
+        runtime=300,
+    shell:
+        execenv_pyexe(config, "build-tier-raw-orca") + "--log {log} "
+        f"--configs {ro(configs)} "
+        f"--chan-maps {ro(chan_maps)} "
+        "--datatype {params.datatype} "
+        "--timestamp {params.timestamp} "
+        "{params.ro_input} {output}"
+
+
 rule build_raw_fcio:
     """
     This rule runs build_raw, it takes in a file.{daq_ext} and outputs a raw file

workflow/src/legenddataflow/FileKey.py

@@ -32,7 +32,12 @@ def regex_from_filepattern(filepattern):
             f.append(f"(?P={wildcard})")
         else:
             wildcards.append(wildcard)
-            f.append(f"(?P<{wildcard}>.+)")
+            if wildcard == "ext":
+                f.append(
+                    f"(?P<{wildcard}>.*)"
+                )  # this means ext will capture everything after 1st dot
+            else:
+                f.append(f"(?P<{wildcard}>" + r"[^\.\/]+)")
         last = match.end()
     f.append(re.escape(filepattern[last:]))
     f.append("$")

workflow/src/legenddataflow/create_pars_keylist.py

@@ -53,9 +53,14 @@ def generate_par_keylist(keys):
 
     @staticmethod
     def match_entries(entry1, entry2):
-        datatype2 = ProcessingFileKey.get_filekey_from_filename(entry2.file[0]).datatype
+        datatype2 = ProcessingFileKey.get_filekey_from_filename(
+            Path(entry2.file[0]).name
+        ).datatype
         for entry in entry1.file:
-            if ProcessingFileKey.get_filekey_from_filename(entry).datatype == datatype2:
+            if (
+                ProcessingFileKey.get_filekey_from_filename(Path(entry).name).datatype
+                == datatype2
+            ):
                 pass
             else:
                 entry2.file.append(entry)