diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f713be..074c838 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,7 +78,7 @@ repos: rev: "v2.3.0" hooks: - id: codespell - args: ["-L", "nd,unparseable,compiletime,livetime,fom,puls"] + args: ["-L", "nd,unparseable,compiletime,livetime,fom,puls,crate"] - repo: https://github.com/shellcheck-py/shellcheck-py rev: "v0.10.0.1" diff --git a/workflow/rules/raw.smk b/workflow/rules/raw.smk index 25f9b37..e001047 100644 --- a/workflow/rules/raw.smk +++ b/workflow/rules/raw.smk @@ -48,6 +48,16 @@ rule build_raw_orca: "{params.ro_input} {output}" +use rule build_raw_orca as build_raw_orca_bz2 with: + input: + get_pattern_tier_daq(config, extension="orca.bz2"), + + +use rule build_raw_orca as build_raw_orca_gz with: + input: + get_pattern_tier_daq(config, extension="orca.gz"), + + rule build_raw_fcio: """ This rule runs build_raw, it takes in a file.{daq_ext} and outputs a raw file diff --git a/workflow/src/legenddataflow/FileKey.py b/workflow/src/legenddataflow/FileKey.py index 6857e05..43162a2 100644 --- a/workflow/src/legenddataflow/FileKey.py +++ b/workflow/src/legenddataflow/FileKey.py @@ -32,7 +32,12 @@ def regex_from_filepattern(filepattern): f.append(f"(?P={wildcard})") else: wildcards.append(wildcard) - f.append(f"(?P<{wildcard}>.+)") + if wildcard == "ext": + f.append( + f"(?P<{wildcard}>.*)" + ) # this means ext will capture everything after 1st dot + else: + f.append(f"(?P<{wildcard}>" + r"[^\.\/]+)") last = match.end() f.append(re.escape(filepattern[last:])) f.append("$") diff --git a/workflow/src/legenddataflow/create_pars_keylist.py b/workflow/src/legenddataflow/create_pars_keylist.py index b779b12..45e1a30 100644 --- a/workflow/src/legenddataflow/create_pars_keylist.py +++ b/workflow/src/legenddataflow/create_pars_keylist.py @@ -53,9 +53,14 @@ def generate_par_keylist(keys): @staticmethod def match_entries(entry1, entry2): - datatype2 = ProcessingFileKey.get_filekey_from_filename(entry2.file[0]).datatype + datatype2 = ProcessingFileKey.get_filekey_from_filename( + Path(entry2.file[0]).name + ).datatype for entry in entry1.file: - if ProcessingFileKey.get_filekey_from_filename(entry).datatype == datatype2: + if ( + ProcessingFileKey.get_filekey_from_filename(Path(entry).name).datatype + == datatype2 + ): pass else: entry2.file.append(entry) diff --git a/workflow/src/legenddataflow/scripts/tier/raw_fcio.py b/workflow/src/legenddataflow/scripts/tier/raw_fcio.py index 67942b9..a351d75 100644 --- a/workflow/src/legenddataflow/scripts/tier/raw_fcio.py +++ b/workflow/src/legenddataflow/scripts/tier/raw_fcio.py @@ -47,4 +47,10 @@ def build_tier_raw_fcio() -> None: if "muon_config" in channel_dict: raise NotImplementedError() - build_raw(args.input, out_spec=all_config, filekey=args.output, **settings) + build_raw( + args.input, + out_spec=all_config, + in_stream_type="Flashcam", + filekey=args.output, + **settings, + ) diff --git a/workflow/src/legenddataflow/scripts/tier/raw_orca.py b/workflow/src/legenddataflow/scripts/tier/raw_orca.py index 9ee5a0f..47ba391 100644 --- a/workflow/src/legenddataflow/scripts/tier/raw_orca.py +++ b/workflow/src/legenddataflow/scripts/tier/raw_orca.py @@ -27,7 +27,7 @@ def build_tier_raw_orca() -> None: configs = TextDB(args.configs, lazy=True) config_dict = configs.on(args.timestamp, system=args.datatype)["snakemake_rules"][ - "tier_raw" + "tier_raw_orca" ] build_log(config_dict, args.log) @@ -63,11 +63,25 @@ def build_tier_raw_orca() -> None: spm_config[next(iter(spm_config))]["spms"]["key_list"] = sorted(spm_channels) Props.add_to(all_config, spm_config) + if "muon_config" in list(channel_dict): + muon_config = Props.read_from(channel_dict["muon_config"]) + muon_channels = list( + chmap.channelmaps.on(args.timestamp) + .map("system", unique=False)["muon"] + .map("daq.rawid") + ) + top_key = next(iter(muon_config)) + muon_config[top_key][next(iter(muon_config[top_key]))]["key_list"] = sorted( + muon_channels + ) + Props.add_to(all_config, muon_config) + if "auxs_config" in list(channel_dict): aux_config = Props.read_from(channel_dict["auxs_config"]) aux_channels = list( chmap.channelmaps.on(args.timestamp) .map("system", unique=False)["auxs"] + .map("daq.crate", unique=False)[1] .map("daq.rawid") ) aux_channels += list( @@ -86,17 +100,10 @@ def build_tier_raw_orca() -> None: ) Props.add_to(all_config, aux_config) - if "muon_config" in list(channel_dict): - muon_config = Props.read_from(channel_dict["muon_config"]) - muon_channels = list( - chmap.channelmaps.on(args.timestamp) - .map("system", unique=False)["muon"] - .map("daq.rawid") - ) - top_key = next(iter(muon_config)) - muon_config[top_key][next(iter(muon_config[top_key]))]["key_list"] = sorted( - muon_channels - ) - Props.add_to(all_config, muon_config) - - build_raw(args.input, out_spec=all_config, filekey=args.output, **settings) + build_raw( + args.input, + out_spec=all_config, + in_stream_type="ORCA", + filekey=args.output, + **settings, + )