Skip to content

Commit 14e523e

Browse files
ggmarshallggmarshall
committed
use channel merge func
1 parent d2a881a commit 14e523e

File tree

6 files changed

+29
-422
lines changed

6 files changed

+29
-422
lines changed

‎workflow/rules/channel_merge.smk

+20-17
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
1-
from scripts.util.patterns import (
1+
from legenddataflow.patterns import (
22
get_pattern_pars_tmp_channel,
33
get_pattern_plts_tmp_channel,
44
get_pattern_plts,
55
get_pattern_tier,
66
get_pattern_pars_tmp,
77
get_pattern_pars,
88
)
9-
from scripts.util.utils import set_last_rule_name
9+
from legenddataflow.utils import set_last_rule_name
1010
import inspect
1111
from legenddataflow.execenv import execenv_smk_py_script
1212

1313
def build_merge_rules(tier,lh5_merge=False):
1414
rule:
1515
input:
1616
lambda wildcards: get_plt_chanlist(
17-
setup,
17+
config,
1818
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
1919
tier,
2020
basedir,
@@ -25,7 +25,7 @@ def build_merge_rules(tier,lh5_merge=False):
2525
timestamp="{timestamp}",
2626
datatype="cal",
2727
output:
28-
get_pattern_plts(setup, tier),
28+
get_pattern_plts(config, tier),
2929
group:
3030
f"merge-{tier}"
3131
shell:
@@ -39,7 +39,7 @@ def build_merge_rules(tier,lh5_merge=False):
3939
rule:
4040
input:
4141
lambda wildcards: get_par_chanlist(
42-
setup,
42+
config,
4343
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
4444
tier,
4545
basedir,
@@ -48,9 +48,12 @@ def build_merge_rules(tier,lh5_merge=False):
4848
name="objects",
4949
extension="pkl",
5050
),
51+
params:
52+
timestamp="{timestamp}",
53+
datatype="cal",
5154
output:
5255
get_pattern_pars(
53-
setup,
56+
config,
5457
tier,
5558
name="objects",
5659
extension="dir",
@@ -71,7 +74,7 @@ def build_merge_rules(tier,lh5_merge=False):
7174
rule:
7275
input:
7376
lambda wildcards: get_par_chanlist(
74-
setup,
77+
config,
7578
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
7679
tier,
7780
basedir,
@@ -84,15 +87,15 @@ def build_merge_rules(tier,lh5_merge=False):
8487
output:
8588
temp(
8689
get_pattern_pars_tmp(
87-
setup,
90+
config,
8891
tier,
8992
datatype="cal",
9093
)
9194
),
9295
group:
9396
f"merge-{tier}"
9497
shell:
95-
execenv_smk_py_script(config, "merge_channels")
98+
f'{execenv_smk_py_script(config, "merge_channels")}'
9699
"--input {input} "
97100
"--output {output} "
98101
"--timestamp {params.timestamp} "
@@ -103,7 +106,7 @@ def build_merge_rules(tier,lh5_merge=False):
103106
rule:
104107
input:
105108
in_files=lambda wildcards: get_par_chanlist(
106-
setup,
109+
config,
107110
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
108111
tier,
109112
basedir,
@@ -112,13 +115,13 @@ def build_merge_rules(tier,lh5_merge=False):
112115
extension="lh5" if lh5_merge is True else inspect.signature(get_par_chanlist).parameters['extension'].default,
113116
),
114117
in_db=get_pattern_pars_tmp(
115-
setup,
118+
config,
116119
"dsp",
117120
datatype="cal",
118-
) if lh5_merge is True else None,
119-
plts=get_pattern_plts(setup, "dsp"),
121+
) if lh5_merge is True else [],
122+
plts=get_pattern_plts(config, "dsp"),
120123
objects=get_pattern_pars(
121-
setup,
124+
config,
122125
"dsp",
123126
name="objects",
124127
extension="dir",
@@ -129,17 +132,17 @@ def build_merge_rules(tier,lh5_merge=False):
129132
datatype="cal",
130133
output:
131134
out_file=get_pattern_pars(
132-
setup,
135+
config,
133136
tier,
134137
extension="lh5" if lh5_merge is True else inspect.signature(get_pattern_pars).parameters['extension'].default,
135138
check_in_cycle=check_in_cycle,
136139
),
137-
out_db=get_pattern_pars(setup, tier, check_in_cycle=check_in_cycle) if lh5_merge is True else None,
140+
out_db=get_pattern_pars(config, tier, check_in_cycle=check_in_cycle) if lh5_merge is True else [],
138141
group:
139142
f"merge-{tier}"
140143
run:
141144
shell_string = (
142-
execenv_smk_py_script(config, "merge_channels")
145+
f'{execenv_smk_py_script(config, "merge_channels")}'
143146
"--output {output.out_file} "
144147
"--input {input.in_files} "
145148
"--timestamp {params.timestamp} "

‎workflow/rules/dsp.smk

+2-132
Original file line numberDiff line numberDiff line change
@@ -30,140 +30,10 @@ Path(dsp_par_cat_file).parent.mkdir(parents=True, exist_ok=True)
3030
ParsKeyResolve.write_to_yaml(dsp_par_catalog, dsp_par_cat_file)
3131

3232

33-
rule build_plts_dsp:
34-
input:
35-
lambda wildcards: get_plt_chanlist(
36-
config,
37-
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
38-
"dsp",
39-
basedir,
40-
det_status,
41-
chan_maps,
42-
),
43-
params:
44-
timestamp="{timestamp}",
45-
datatype="cal",
46-
output:
47-
get_pattern_plts(config, "dsp"),
48-
group:
49-
"merge-dsp"
50-
shell:
51-
f'{execenv_smk_py_script(config, "merge_channels")}'
52-
"--input {input} "
53-
"--output {output} "
54-
"--channelmap {meta} "
33+
include: "channel_merge.smk"
5534

5635

57-
rule build_pars_dsp_objects:
58-
input:
59-
lambda wildcards: get_par_chanlist(
60-
config,
61-
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
62-
"dsp",
63-
basedir,
64-
det_status,
65-
chan_maps,
66-
name="objects",
67-
extension="pkl",
68-
),
69-
params:
70-
timestamp="{timestamp}",
71-
datatype="cal",
72-
output:
73-
get_pattern_pars(
74-
config,
75-
"dsp",
76-
name="objects",
77-
extension="dir",
78-
check_in_cycle=check_in_cycle,
79-
),
80-
group:
81-
"merge-dsp"
82-
shell:
83-
f'{execenv_smk_py_script(config, "merge_channels")}'
84-
"--input {input} "
85-
"--output {output} "
86-
"--timestamp {params.timestamp} "
87-
"--channelmap {meta} "
88-
89-
90-
rule build_pars_dsp_db:
91-
input:
92-
lambda wildcards: get_par_chanlist(
93-
config,
94-
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
95-
"dsp",
96-
basedir,
97-
det_status,
98-
chan_maps,
99-
),
100-
params:
101-
timestamp="{timestamp}",
102-
datatype="cal",
103-
output:
104-
temp(
105-
get_pattern_pars_tmp(
106-
config,
107-
"dsp",
108-
datatype="cal",
109-
)
110-
),
111-
group:
112-
"merge-dsp"
113-
shell:
114-
f'{execenv_smk_py_script(config, "merge_channels")}'
115-
"--input {input} "
116-
"--output {output} "
117-
"--timestamp {params.timestamp} "
118-
"--channelmap {meta} "
119-
120-
121-
rule build_pars_dsp:
122-
input:
123-
in_files=lambda wildcards: get_par_chanlist(
124-
config,
125-
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
126-
"dsp",
127-
basedir,
128-
det_status,
129-
chan_maps,
130-
name="dplms",
131-
extension="lh5",
132-
),
133-
in_db=get_pattern_pars_tmp(
134-
config,
135-
"dsp",
136-
datatype="cal",
137-
),
138-
plts=get_pattern_plts(config, "dsp"),
139-
objects=get_pattern_pars(
140-
config,
141-
"dsp",
142-
name="objects",
143-
extension="dir",
144-
check_in_cycle=check_in_cycle,
145-
),
146-
params:
147-
timestamp="{timestamp}",
148-
datatype="cal",
149-
output:
150-
out_file=get_pattern_pars(
151-
config,
152-
"dsp",
153-
extension="lh5",
154-
check_in_cycle=check_in_cycle,
155-
),
156-
out_db=get_pattern_pars(config, "dsp", check_in_cycle=check_in_cycle),
157-
group:
158-
"merge-dsp"
159-
shell:
160-
f'{execenv_smk_py_script(config, "merge_channels")}'
161-
"--output {output.out_file} "
162-
"--in_db {input.in_db} "
163-
"--out_db {output.out_db} "
164-
"--input {input.in_files} "
165-
"--timestamp {params.timestamp} "
166-
"--channelmap {meta} "
36+
build_merge_rules("dsp", lh5_merge=True)
16737

16838

16939
rule build_dsp:

‎workflow/rules/dsp_pars_geds.smk

+1-3
Original file line numberDiff line numberDiff line change
@@ -141,9 +141,7 @@ rule build_pars_dsp_dplms_geds:
141141
channel="{channel}",
142142
output:
143143
dsp_pars=temp(get_pattern_pars_tmp_channel(config, "dsp", "dplms")),
144-
lh5_path=temp(
145-
get_pattern_pars_tmp_channel(config, "dsp", "dplms", extension="lh5")
146-
),
144+
lh5_path=temp(get_pattern_pars_tmp_channel(config, "dsp", extension="lh5")),
147145
plots=temp(get_pattern_plts_tmp_channel(config, "dsp", "dplms")),
148146
log:
149147
get_pattern_log_channel(config, "pars_dsp_dplms", time),

‎workflow/rules/hit.smk

+2-81
Original file line numberDiff line numberDiff line change
@@ -219,89 +219,10 @@ rule build_lq_calibration:
219219
"{input.files}"
220220

221221

222-
rule build_pars_hit_objects:
223-
input:
224-
lambda wildcards: get_par_chanlist(
225-
config,
226-
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
227-
"hit",
228-
basedir,
229-
det_status,
230-
chan_maps,
231-
name="objects",
232-
extension="pkl",
233-
),
234-
output:
235-
get_pattern_pars(
236-
config,
237-
"hit",
238-
name="objects",
239-
extension="dir",
240-
check_in_cycle=check_in_cycle,
241-
),
242-
params:
243-
ro_input=lambda _, input: ro(input),
244-
group:
245-
"merge-hit"
246-
shell:
247-
f'{execenv_smk_py_script(config, "merge_channels")}'
248-
"--input {params.ro_input} "
249-
"--output {output} "
250-
"--channelmap {meta} "
222+
include: "channel_merge.smk"
251223

252224

253-
rule build_plts_hit:
254-
input:
255-
lambda wildcards: get_plt_chanlist(
256-
config,
257-
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
258-
"hit",
259-
basedir,
260-
det_status,
261-
chan_maps,
262-
),
263-
output:
264-
get_pattern_plts(config, "hit"),
265-
params:
266-
ro_input=lambda _, input: ro(input),
267-
group:
268-
"merge-hit"
269-
shell:
270-
f'{execenv_smk_py_script(config, "merge_channels")}'
271-
"--input {params.ro_input} "
272-
"--output {output} "
273-
"--channelmap {meta} "
274-
275-
276-
rule build_pars_hit:
277-
input:
278-
infiles=lambda wildcards: get_par_chanlist(
279-
config,
280-
f"all-{wildcards.experiment}-{wildcards.period}-{wildcards.run}-cal-{wildcards.timestamp}-channels",
281-
"hit",
282-
basedir,
283-
det_status,
284-
chan_maps,
285-
),
286-
plts=get_pattern_plts(config, "hit"),
287-
objects=get_pattern_pars(
288-
config,
289-
"hit",
290-
name="objects",
291-
extension="dir",
292-
check_in_cycle=check_in_cycle,
293-
),
294-
params:
295-
ro_input=lambda _, input: {k: ro(v) for k, v in input.items()},
296-
output:
297-
get_pattern_pars(config, "hit", check_in_cycle=check_in_cycle),
298-
group:
299-
"merge-hit"
300-
shell:
301-
f'{execenv_smk_py_script(config, "merge_channels")}'
302-
"--input {params.ro_input[infiles]} "
303-
"--output {output} "
304-
"--channelmap {meta} "
225+
build_merge_rules("hit", lh5_merge=False)
305226

306227

307228
rule build_hit:

0 commit comments

Comments
 (0)