bugfixes round1

Author: ggmarshall

workflow/Snakefile

@@ -20,8 +20,7 @@ import logging
 
 from dbetto import AttrsDict
 from legendmeta import LegendMetadata
-from legenddataflow import CalGrouping
-from legenddataflow import utils
+from legenddataflow import CalGrouping, execenv, utils
 
 utils.subst_vars_in_snakemake_config(workflow, config)
 config = AttrsDict(config)
@@ -38,6 +37,8 @@ time = datetime.now().strftime("%Y%m%dT%H%M%SZ")
 if not Path(meta).exists():
     LegendMetadata(meta).checkout(config.legend_metadata_version)
 
+metadata = LegendMetadata(meta, lazy=True)
+
 part = CalGrouping(config, Path(det_status) / "cal_groupings.yaml")
 
 
@@ -80,7 +81,7 @@ onstart:
 
     # Make sure some packages are initialized before we begin to avoid race conditions
     for pkg in ["dspeed", "lgdo", "matplotlib"]:
-        shell(execenv.execenv_pyexe(config, "python") + f" -c 'import {pkg}'")
+        shell(execenv.execenv_pyexe(config, "python") + "-c 'import " + pkg + "'")
 
         # Log parameter catalogs in validity.jsonl files
     hit_par_cat_file = Path(utils.pars_path(config)) / "hit" / "validity.yaml"
@@ -123,9 +124,11 @@ onstart:
 onsuccess:
     from snakemake.report import auto_report
 
+
     rep_dir = f"{log_path(config)}/report-{datetime.strftime(datetime.utcnow(), '%Y%m%dT%H%M%SZ')}"
     Path(rep_dir).mkdir(parents=True, exist_ok=True)
     # auto_report(workflow.persistence.dag, f"{rep_dir}/report.html")
+    auto_report(workflow.persistence.dag, report_plugin, report_settings)
 
     with open(os.path.join(rep_dir, "dag.txt"), "w") as f:
         f.writelines(str(workflow.persistence.dag))

workflow/profiles/lngs/config.yaml

@@ -1,6 +1,5 @@
 cores: 125
 restart-times: 2
-max-jobs-per-second: 1
 resources:
   - mem_swap=3500
 configfile: config-lngs.yaml

workflow/src/legenddataflow/scripts/par/geds/dsp/dplms.py

@@ -10,8 +10,8 @@
 from lgdo import Array, Table
 from pygama.pargen.dplms_ge_dict import dplms_ge_dict
 
-from ....log import build_log
-from ...table_name import get_table_name
+from .....log import build_log
+from ....table_name import get_table_name
 
 
 def par_geds_dsp_dplms() -> None:

workflow/src/legenddataflow/scripts/par/geds/dsp/eopt.py

@@ -18,8 +18,8 @@
     run_one_dsp,
 )
 
-from ....log import build_log
-from ...table_name import get_table_name
+from .....log import build_log
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 warnings.filterwarnings(action="ignore", category=np.RankWarning)

workflow/src/legenddataflow/scripts/par/geds/dsp/evtsel.py

@@ -15,9 +15,9 @@
 from pygama.pargen.data_cleaning import generate_cuts, get_keys
 from pygama.pargen.dsp_optimize import run_one_dsp
 
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from .....log import build_log
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

workflow/src/legenddataflow/scripts/par/geds/dsp/nopt.py

@@ -12,7 +12,7 @@
 from pygama.pargen.data_cleaning import generate_cuts, get_cut_indexes
 from pygama.pargen.dsp_optimize import run_one_dsp
 
-from ....log import build_log
+from .....log import build_log
 
 
 def par_geds_dsp_nopt() -> None:

workflow/src/legenddataflow/scripts/par/geds/dsp/svm_build.py

@@ -7,7 +7,7 @@
 from lgdo import lh5
 from sklearn.svm import SVC
 
-from ....log import build_log
+from .....log import build_log
 
 
 def par_geds_dsp_svm_build() -> None:

workflow/src/legenddataflow/scripts/par/geds/dsp/tau.py

@@ -10,9 +10,9 @@
 from pygama.pargen.dsp_optimize import run_one_dsp
 from pygama.pargen.extract_tau import ExtractTau
 
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from .....log import build_log
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 
 def par_geds_dsp_tau() -> None:

workflow/src/legenddataflow/scripts/par/geds/hit/aoe.py

@@ -12,10 +12,10 @@
 from pygama.pargen.AoE_cal import CalAoE, Pol1, SigmaFit, aoe_peak
 from pygama.pargen.utils import load_data
 
+from .....log import build_log
 from ....convert_np import convert_dict_np_to_float
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

workflow/src/legenddataflow/scripts/par/geds/hit/ecal.py

@@ -23,10 +23,10 @@
 from pygama.pargen.utils import load_data
 from scipy.stats import binned_statistic
 
+from .....log import build_log
 from ....convert_np import convert_dict_np_to_float
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 mpl.use("agg")
 sto = lh5.LH5Store()

workflow/src/legenddataflow/scripts/par/geds/hit/lq.py

@@ -14,10 +14,10 @@
 from pygama.pargen.lq_cal import LQCal
 from pygama.pargen.utils import load_data
 
+from .....log import build_log
 from ....convert_np import convert_dict_np_to_float
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

workflow/src/legenddataflow/scripts/par/geds/hit/qc.py

@@ -17,10 +17,10 @@
 )
 from pygama.pargen.utils import load_data
 
+from .....log import build_log
 from ....convert_np import convert_dict_np_to_float
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

workflow/src/legenddataflow/scripts/par/geds/pht/aoe.py

@@ -16,10 +16,10 @@
 from pygama.pargen.AoE_cal import CalAoE, Pol1, SigmaFit, aoe_peak
 from pygama.pargen.utils import load_data
 
+from .....log import build_log
 from ....FileKey import ChannelProcKey, ProcessingFileKey
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

workflow/src/legenddataflow/scripts/par/geds/pht/ecal_part.py

@@ -18,10 +18,10 @@
 from pygama.pargen.energy_cal import FWHMLinear, FWHMQuadratic, HPGeCalibration
 from pygama.pargen.utils import load_data
 
+from .....log import build_log
 from ....FileKey import ChannelProcKey, ProcessingFileKey
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 warnings.filterwarnings(action="ignore", category=np.RankWarning)

workflow/src/legenddataflow/scripts/par/geds/pht/fast.py

@@ -16,10 +16,10 @@
 from workflow.src.legenddataflow.scripts.par.geds.pht.lq import run_lq_calibration
 from workflow.src.legenddataflow.scripts.par.geds.pht.partcal import calibrate_partition
 
+from .....log import build_log
 from ....FileKey import ChannelProcKey, ProcessingFileKey
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 warnings.filterwarnings(action="ignore", category=np.RankWarning)

workflow/src/legenddataflow/scripts/par/geds/pht/lq.py

@@ -18,10 +18,10 @@
 from pygama.pargen.lq_cal import LQCal
 from pygama.pargen.utils import load_data
 
+from .....log import build_log
 from ....FileKey import ChannelProcKey, ProcessingFileKey
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

workflow/src/legenddataflow/scripts/par/geds/pht/qc.py

@@ -17,10 +17,10 @@
 )
 from pygama.pargen.utils import load_data
 
+from .....log import build_log
 from ....convert_np import convert_dict_np_to_float
-from ....log import build_log
-from ...pulser_removal import get_pulser_mask
-from ...table_name import get_table_name
+from ....pulser_removal import get_pulser_mask
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

workflow/src/legenddataflow/scripts/par/geds/pht/qc_phy.py

@@ -17,9 +17,9 @@
     get_keys,
 )
 
+from .....log import build_log
 from ....convert_np import convert_dict_np_to_float
-from ....log import build_log
-from ...table_name import get_table_name
+from ....table_name import get_table_name
 
 warnings.filterwarnings(action="ignore", category=RuntimeWarning)
 

workflow/src/legenddataflow/scripts/par/geds/raw/blindcal.py

@@ -16,7 +16,7 @@
 from lgdo import lh5
 from pygama.pargen.energy_cal import HPGeCalibration
 
-from ...table_name import get_table_name
+from ....table_name import get_table_name
 
 mpl.use("agg")
 

workflow/src/legenddataflow/scripts/par/geds/raw/blindcheck.py

@@ -21,8 +21,8 @@
 from pygama.math.histogram import get_hist
 from pygama.pargen.energy_cal import get_i_local_maxima
 
-from ....log import build_log
-from ...table_name import get_table_name
+from .....log import build_log
+from ....table_name import get_table_name
 
 mpl.use("Agg")
 

workflow/src/legenddataflow/scripts/par/geds/tcm/pulser.py

@@ -6,8 +6,8 @@
 from dbetto.catalog import Props
 from pygama.pargen.data_cleaning import get_tcm_pulser_ids
 
-from ....log import build_log
-from ...table_name import get_table_name
+from .....log import build_log
+from ....table_name import get_table_name
 
 
 def par_geds_tcm_pulser() -> None:

workflow/src/legenddataflow/scripts/pulser_removal.py

@@ -1,30 +1,14 @@
-from pathlib import Path
-
 import numpy as np
 from dbetto.catalog import Props
-from pygama.pargen.data_cleaning import get_tcm_pulser_ids
-
 
-def get_pulser_mask(
-    pulser_file=None, tcm_filelist=None, channel=None, pulser_multiplicity_threshold=10
-):
-    if pulser_file is not None:
-        if not isinstance(pulser_file, list):
-            pulser_file = [pulser_file]
-        mask = np.array([], dtype=bool)
-        for file in pulser_file:
-            pulser_dict = Props.read_from(file)
-            pulser_mask = np.array(pulser_dict["mask"])
-            mask = np.append(mask, pulser_mask)
 
-    elif tcm_filelist is not None:
-        # get pulser mask from tcm files
-        with Path(tcm_filelist).open() as f:
-            tcm_files = f.read().splitlines()
-        tcm_files = sorted(np.unique(tcm_files))
-        _, mask = get_tcm_pulser_ids(tcm_files, channel, pulser_multiplicity_threshold)
-    else:
-        msg = "No pulser file or tcm filelist provided"
-        raise ValueError(msg)
+def get_pulser_mask(pulser_file):
+    if not isinstance(pulser_file, list):
+        pulser_file = [pulser_file]
+    mask = np.array([], dtype=bool)
+    for file in pulser_file:
+        pulser_dict = Props.read_from(file)
+        pulser_mask = np.array(pulser_dict["mask"])
+        mask = np.append(mask, pulser_mask)
 
     return mask