From ff159f55ff7024c225e0a86464dfaf25a38cece1 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Mon, 18 Sep 2023 13:52:28 +0000 Subject: [PATCH] Bugs + minor fixes --- nsb_scripts/LSTnsb.py | 4 ++-- nsb_scripts/nsb_coincident_events.py | 4 ++-- nsb_scripts/nsb_level.py | 11 ++++++----- nsb_scripts/nsb_merge_M1_M2_night.py | 6 +----- nsb_scripts/nsb_merge_M1_M2_runs.py | 27 +++++++++++++-------------- nsb_scripts/nsb_merge_subruns.py | 2 +- nsb_scripts/nsb_stereo_events.py | 5 +++-- 7 files changed, 28 insertions(+), 31 deletions(-) diff --git a/nsb_scripts/LSTnsb.py b/nsb_scripts/LSTnsb.py index 6237cc27f..1129a55c4 100644 --- a/nsb_scripts/LSTnsb.py +++ b/nsb_scripts/LSTnsb.py @@ -82,13 +82,13 @@ def main(): noise.append(a) a = sum(noise) / len(noise) std = np.std(noise) - logger.info("run", run_number, "nsb average (all)", a, "std", std) + logger.info(f"Run n. {run_number}", "nsb average (all)", a, "std", std) subrun_ok = [] for sr in range(0, len(noise)): if np.abs(noise[sr] - a) < 3 * std: subrun_ok.append(noise[sr]) a = sum(subrun_ok) / len(subrun_ok) - logger.info("run", run_number, "nsb average (w/o outliers)", a) + logger.info(f"Run n. {run_number}", "nsb average (w/o outliers)", a) for j in range(0, len(nsb)): if (a < nsb_limit[j + 1]) & (a > nsb_limit[j]): with open(f"{source}_LST_{nsb[j]}_{run_number}.txt", "a+") as f: diff --git a/nsb_scripts/nsb_coincident_events.py b/nsb_scripts/nsb_coincident_events.py index ba3b608e4..32901a22a 100644 --- a/nsb_scripts/nsb_coincident_events.py +++ b/nsb_scripts/nsb_coincident_events.py @@ -60,7 +60,7 @@ def configfile_coincidence(ids, target_dir): + "\n\n" ) f.write( - 'event_coincidence:\n timestamp_type_lst: "dragon_time" # select "dragon_time", "tib_time" or "ucts_time"\n window_half_width: "300 ns"\n' + 'event_coincidence:\n timestamp_type_lst: "dragon_time" # select "dragon_time", "tib_time" or "ucts_time"\n pre_offset_search: true\n n_pre_offset_search_events: 100\n window_half_width: "300 ns"\n' ) f.write(' time_offset:\n start: "-10 us"\n stop: "0 us"\n') f.close() @@ -213,7 +213,7 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date): + str(nsb) + "\n" ) - f.write(f"#SBATCH --array=0-{process_size}%50\n") + f.write(f"#SBATCH --array=0-{process_size}\n") f.write("#SBATCH --mem=30g\n") f.write("#SBATCH -N 1\n\n") f.write("ulimit -l unlimited\n") diff --git a/nsb_scripts/nsb_level.py b/nsb_scripts/nsb_level.py index eaa402e4b..d26fe5add 100644 --- a/nsb_scripts/nsb_level.py +++ b/nsb_scripts/nsb_level.py @@ -14,7 +14,8 @@ logger.setLevel(logging.INFO) -def bash_scripts(run, config): +def bash_scripts(run, config, source): + f = open(f"run_{run}.sh", "w") f.write("#!/bin/sh\n\n") f.write("#SBATCH -p long\n") @@ -26,7 +27,7 @@ def bash_scripts(run, config): f.write("ulimit -a\n\n") f.write( - f"conda run -n magic-lst python LSTnsb.py -c {config} -i {run} > nsblog_{run}.log 2>&1 \n\n" + f"conda run -n magic-lst python LSTnsb.py -c {config} -i {run} > {source}_nsblog_{run}.log 2>&1 \n\n" ) f.close() @@ -47,15 +48,15 @@ def main(): args.config_file, "rb" ) as f: # "rb" mode opens the file in binary format for reading config = yaml.safe_load(f) - + source=config["directories"]["target_name"] runs = config["general"]["LST_runs"] with open(str(runs), "r") as LSTfile: run = LSTfile.readlines() - print("***** Generating merge bashscripts...") + print("***** Generating bashscripts...") for i in run: i = i.rstrip() - bash_scripts(i, args.config_file) + bash_scripts(i, args.config_file, source) print("Process name: nsb") print( "To check the jobs submitted to the cluster, type: squeue -n nsb" diff --git a/nsb_scripts/nsb_merge_M1_M2_night.py b/nsb_scripts/nsb_merge_M1_M2_night.py index fedb51640..81a109dce 100644 --- a/nsb_scripts/nsb_merge_M1_M2_night.py +++ b/nsb_scripts/nsb_merge_M1_M2_night.py @@ -41,11 +41,7 @@ def merge(scripts_dir, target_dir): process_name = "merging_" + target_dir.split("/")[-2:][1] MAGIC_DL1_dir = target_dir + f"/v{__version__}" + "/DL1/" + p - if os.path.exists(MAGIC_DL1_dir + "/M1") & os.path.exists( - MAGIC_DL1_dir + "/M2" - ): - if not os.path.exists(MAGIC_DL1_dir + "/Merged"): - os.mkdir(MAGIC_DL1_dir + "/Merged") + f = open(f"Merge_2_{p}.sh", "w") f.write("#!/bin/sh\n\n") diff --git a/nsb_scripts/nsb_merge_M1_M2_runs.py b/nsb_scripts/nsb_merge_M1_M2_runs.py index 02be425db..adabe4259 100644 --- a/nsb_scripts/nsb_merge_M1_M2_runs.py +++ b/nsb_scripts/nsb_merge_M1_M2_runs.py @@ -41,11 +41,7 @@ def merge(scripts_dir, target_dir): process_name = "merging_" + target_dir.split("/")[-2:][1] MAGIC_DL1_dir = target_dir + f"/v{__version__}" + "/DL1/" + p - if os.path.exists(MAGIC_DL1_dir + "/M1") & os.path.exists( - MAGIC_DL1_dir + "/M2" - ): - if not os.path.exists(MAGIC_DL1_dir + "/Merged"): - os.mkdir(MAGIC_DL1_dir + "/Merged") + f = open(f"Merge_1_{p}.sh", "w") f.write("#!/bin/sh\n\n") @@ -69,15 +65,18 @@ def merge(scripts_dir, target_dir): for x in glob.glob(f"{MAGIC_DL1_dir}/M2/{i}/*") ] for r in runs: - if not os.path.exists(MAGIC_DL1_dir + f"/Merged/{i}/Merged"): - os.mkdir(f"{MAGIC_DL1_dir}/Merged/{i}/Merged") - if not os.path.exists( - MAGIC_DL1_dir + f"/Merged/{i}/Merged/logs" - ): - os.mkdir(f"{MAGIC_DL1_dir}/Merged/{i}/Merged/logs") - f.write( - f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --run-wise >{MAGIC_DL1_dir}/Merged/{i}/Merged/logs/merge_{i}.log \n" - ) + + if (len(os.listdir(MAGIC_DL1_dir + f"/M1/{i}/{r}"))>1) and (len(os.listdir(MAGIC_DL1_dir + f"/M2/{i}/{r}")))>1: + + if not os.path.exists(MAGIC_DL1_dir + f"/Merged/{i}/Merged"): + os.mkdir(f"{MAGIC_DL1_dir}/Merged/{i}/Merged") + if not os.path.exists( + MAGIC_DL1_dir + f"/Merged/{i}/Merged/logs" + ): + os.mkdir(f"{MAGIC_DL1_dir}/Merged/{i}/Merged/logs") + f.write( + f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --run-wise >{MAGIC_DL1_dir}/Merged/{i}/Merged/logs/merge_{i}.log \n" + ) f.close() diff --git a/nsb_scripts/nsb_merge_subruns.py b/nsb_scripts/nsb_merge_subruns.py index d01d4d017..4f3154635 100644 --- a/nsb_scripts/nsb_merge_subruns.py +++ b/nsb_scripts/nsb_merge_subruns.py @@ -41,12 +41,12 @@ def merge(scripts_dir, target_dir): process_name = "merging_" + target_dir.split("/")[-2:][1] MAGIC_DL1_dir = target_dir + f"/v{__version__}" + "/DL1/" + p + if os.path.exists(MAGIC_DL1_dir + "/M1") & os.path.exists( MAGIC_DL1_dir + "/M2" ): if not os.path.exists(MAGIC_DL1_dir + "/Merged"): os.mkdir(MAGIC_DL1_dir + "/Merged") - f = open(f"Merge_0_{p}.sh", "w") f.write("#!/bin/sh\n\n") f.write("#SBATCH -p short\n") diff --git a/nsb_scripts/nsb_stereo_events.py b/nsb_scripts/nsb_stereo_events.py index 58ecfff53..7dfd3913d 100644 --- a/nsb_scripts/nsb_stereo_events.py +++ b/nsb_scripts/nsb_stereo_events.py @@ -128,7 +128,8 @@ def bash_stereo(scripts_dir, target_dir, nsb): os.mkdir(stereoDir + "/logs") if not os.listdir(f"{nightLST}"): continue - + if len(os.listdir(nightLST ))<2: + continue os.system( f"ls {nightLST}/*LST*.h5 > {stereoDir}/logs/list_coin_{nsb}.txt" ) # generating a list with the DL1 coincident data files. @@ -142,7 +143,7 @@ def bash_stereo(scripts_dir, target_dir, nsb): f.write("#!/bin/sh\n\n") f.write("#SBATCH -p short\n") f.write("#SBATCH -J " + process_name + "_stereo" + str(nsb) + "\n") - f.write(f"#SBATCH --array=0-{process_size}%100\n") + f.write(f"#SBATCH --array=0-{process_size}\n") f.write("#SBATCH -N 1\n\n") f.write("ulimit -l unlimited\n") f.write("ulimit -s unlimited\n")