diff --git a/nsb_scripts/LSTnsb.py b/nsb_scripts/LSTnsb.py index 16bb8ea45..8a7010e1b 100644 --- a/nsb_scripts/LSTnsb.py +++ b/nsb_scripts/LSTnsb.py @@ -79,6 +79,8 @@ def main(): if ii % mod == 0: a, b, c = calculate_noise_parameters(simtel, run_list[ii], lst_config) noise.append(a) + if len(noise)==0: + return a = sum(noise) / len(noise) std = np.std(noise) logger.info(f"Run n. {run_number}, nsb average (all) {a}, std {std}") @@ -86,6 +88,8 @@ def main(): for sr in range(0, len(noise)): if np.abs(noise[sr] - a) < 3 * std: subrun_ok.append(noise[sr]) + if len(subrun_ok)==0: + return a = sum(subrun_ok) / len(subrun_ok) logger.info(f"Run n. {run_number}, nsb average (w/o outliers) {a}") for j in range(0, len(nsb)): diff --git a/nsb_scripts/nsb_coincident_events.py b/nsb_scripts/nsb_coincident_events.py index beb08ae15..19e35365b 100644 --- a/nsb_scripts/nsb_coincident_events.py +++ b/nsb_scripts/nsb_coincident_events.py @@ -220,7 +220,7 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date, source): f.write( "#SBATCH -J " + target_dir.split("/")[-2:][1] - + "_coincidence" + + "_coincidence_" + str(nsb) + "\n" ) @@ -241,7 +241,7 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date, source): "export LOG=$OUTPUTDIR/logs/coincidence_${SLURM_ARRAY_TASK_ID}.log\n" ) f.write( - f"conda run -n magic-lst python {scripts_dir}/lst1_magic_event_coincidence.py --input-file-lst $SAMPLE --input-dir-magic $INM --output-dir $OUTPUTDIR --config-file {target_dir}/config_coincidence.yaml >$LOG 2>&1" + f"time conda run -n magic-lst python {scripts_dir}/lst1_magic_event_coincidence.py --input-file-lst $SAMPLE --input-dir-magic $INM --output-dir $OUTPUTDIR --config-file {target_dir}/config_coincidence.yaml >$LOG 2>&1" ) f.close() @@ -298,13 +298,13 @@ def main(): print( "Process name: " + target_dir.split("/")[-2:][1] - + "_coincidence" + + "_coincidence_" + str(nsblvl) ) print( "To check the jobs submitted to the cluster, type: squeue -n " + target_dir.split("/")[-2:][1] - + "_coincidence" + + "_coincidence_" + str(nsblvl) ) diff --git a/nsb_scripts/nsb_level.py b/nsb_scripts/nsb_level.py index cbedd216d..f5818d3bb 100644 --- a/nsb_scripts/nsb_level.py +++ b/nsb_scripts/nsb_level.py @@ -26,7 +26,7 @@ def bash_scripts(run, config, source): f.write("ulimit -a\n\n") f.write( - f"conda run -n magic-lst python LSTnsb.py -c {config} -i {run} > {source}_nsblog_{run}.log 2>&1 \n\n" + f"time conda run -n magic-lst python LSTnsb.py -c {config} -i {run} > {source}_nsblog_{run}.log 2>&1 \n\n" ) f.close() diff --git a/nsb_scripts/nsb_merge_M1_M2_night.py b/nsb_scripts/nsb_merge_M1_M2_night.py index 3c36bd3ae..f89b594be 100644 --- a/nsb_scripts/nsb_merge_M1_M2_night.py +++ b/nsb_scripts/nsb_merge_M1_M2_night.py @@ -62,7 +62,7 @@ def merge(scripts_dir, target_dir, source): if not os.path.exists(MAGIC_DL1_dir + f"/Merged/Merged_{i}/logs"): os.mkdir(f"{MAGIC_DL1_dir}/Merged/Merged_{i}/logs") f.write( - f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --output-dir {MAGIC_DL1_dir}/Merged/Merged_{i} >{MAGIC_DL1_dir}/Merged/Merged_{i}/logs/merge_night_{i}.log \n" + f"time conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --output-dir {MAGIC_DL1_dir}/Merged/Merged_{i} >{MAGIC_DL1_dir}/Merged/Merged_{i}/logs/merge_night_{i}.log \n" ) f.close() diff --git a/nsb_scripts/nsb_merge_M1_M2_runs.py b/nsb_scripts/nsb_merge_M1_M2_runs.py index 12d654fc8..1245cdcc8 100644 --- a/nsb_scripts/nsb_merge_M1_M2_runs.py +++ b/nsb_scripts/nsb_merge_M1_M2_runs.py @@ -70,7 +70,7 @@ def merge(scripts_dir, target_dir, source): ): os.mkdir(f"{MAGIC_DL1_dir}/Merged/{i}/Merged/logs") f.write( - f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --run-wise >{MAGIC_DL1_dir}/Merged/{i}/Merged/logs/merge_{i}.log \n" + f"time conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --run-wise >{MAGIC_DL1_dir}/Merged/{i}/Merged/logs/merge_{i}.log \n" ) f.close() diff --git a/nsb_scripts/nsb_merge_subruns.py b/nsb_scripts/nsb_merge_subruns.py index 3ae1be373..17bafdfd1 100644 --- a/nsb_scripts/nsb_merge_subruns.py +++ b/nsb_scripts/nsb_merge_subruns.py @@ -78,7 +78,7 @@ def merge(scripts_dir, target_dir, source): ) # Creating a merged directory for the respective run f.write( - f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/M1/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/{r} >{MAGIC_DL1_dir}/Merged/{i}/{r}/logs/merge_M1_{i}_{r}.log \n" + f"time conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/M1/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/{r} >{MAGIC_DL1_dir}/Merged/{i}/{r}/logs/merge_M1_{i}_{r}.log \n" ) if os.path.exists(MAGIC_DL1_dir + "/M2"): @@ -103,7 +103,7 @@ def merge(scripts_dir, target_dir, source): ) # Creating a merged directory for the respective run f.write( - f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/M2/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/{r} >{MAGIC_DL1_dir}/Merged/{i}/{r}/logs/merge_M2_{i}_{r}.log \n" + f"time conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/M2/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/{r} >{MAGIC_DL1_dir}/Merged/{i}/{r}/logs/merge_M2_{i}_{r}.log \n" ) f.close() diff --git a/nsb_scripts/nsb_setting_up_config_and_dir.py b/nsb_scripts/nsb_setting_up_config_and_dir.py index 2540afc49..47bcb437f 100644 --- a/nsb_scripts/nsb_setting_up_config_and_dir.py +++ b/nsb_scripts/nsb_setting_up_config_and_dir.py @@ -98,7 +98,7 @@ def lists_and_bash_gen_MAGIC( Below we create a bash script that links the MAGIC data paths to each subdirectory. """ - process_name = target_dir.split("/")[-2:][0] + target_dir.split("/")[-2:][1] + process_name = target_dir.split("/")[-2:][0] +"_"+ target_dir.split("/")[-2:][1] f = open(f"{source}_linking_MAGIC_data_paths.sh", "w") f.write("#!/bin/sh\n\n") @@ -231,7 +231,7 @@ def lists_and_bash_gen_MAGIC( "export LOG=$OUTPUTDIR/logs/real_0_1_task${SLURM_ARRAY_TASK_ID}.log\n" ) f.write( - f"conda run -n magic-lst python {scripts_dir}/magic_calib_to_dl1.py --input-file $SAMPLE --output-dir $OUTPUTDIR --config-file " + f"time conda run -n magic-lst python {scripts_dir}/magic_calib_to_dl1.py --input-file $SAMPLE --output-dir $OUTPUTDIR --config-file " + target_dir + "/config_step1.yaml >$LOG 2>&1\n" ) @@ -280,7 +280,7 @@ def lists_and_bash_gen_MAGIC( "export LOG=$OUTPUTDIR/logs/real_0_1_task${SLURM_ARRAY_TASK_ID}.log\n" ) f.write( - f"conda run -n magic-lst python {scripts_dir}/magic_calib_to_dl1.py --input-file $SAMPLE --output-dir $OUTPUTDIR --config-file " + f"time conda run -n magic-lst python {scripts_dir}/magic_calib_to_dl1.py --input-file $SAMPLE --output-dir $OUTPUTDIR --config-file " + target_dir + "/config_step1.yaml >$LOG 2>&1\n" ) @@ -388,11 +388,11 @@ def main(): print("*** Reducing DL0 to DL1 data***") print( - "Process name: ", target_dir.split("/")[-2:][0] + target_dir.split("/")[-2:][1] + "Process name: ", target_dir.split("/")[-2:][0] +"_"+ target_dir.split("/")[-2:][1] ) print( "To check the jobs submitted to the cluster, type: squeue -n", - target_dir.split("/")[-2:][0] + target_dir.split("/")[-2:][1], + target_dir.split("/")[-2:][0] +"_"+ target_dir.split("/")[-2:][1], ) directories_generator( diff --git a/nsb_scripts/nsb_stereo_events.py b/nsb_scripts/nsb_stereo_events.py index 856674e89..f11d105fa 100644 --- a/nsb_scripts/nsb_stereo_events.py +++ b/nsb_scripts/nsb_stereo_events.py @@ -145,7 +145,7 @@ def bash_stereo(scripts_dir, target_dir, nsb, source): f = open(f"{source}_StereoEvents_{nsb}_{nightLST.split('/')[-1]}.sh", "w") f.write("#!/bin/sh\n\n") f.write("#SBATCH -p short\n") - f.write("#SBATCH -J " + process_name + "_stereo" + str(nsb) + "\n") + f.write("#SBATCH -J " + process_name + "_stereo_" + str(nsb) + "\n") f.write(f"#SBATCH --array=0-{process_size}\n") f.write("#SBATCH -N 1\n\n") f.write("ulimit -l unlimited\n") @@ -158,7 +158,7 @@ def bash_stereo(scripts_dir, target_dir, nsb, source): f.write("SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n") f.write("export LOG=$OUTPUTDIR/logs/stereo_${SLURM_ARRAY_TASK_ID}.log\n") f.write( - f"conda run -n magic-lst python {scripts_dir}/lst1_magic_stereo_reco.py --input-file $SAMPLE --output-dir $OUTPUTDIR --config-file {target_dir}/config_stereo.yaml >$LOG 2>&1" + f"time conda run -n magic-lst python {scripts_dir}/lst1_magic_stereo_reco.py --input-file $SAMPLE --output-dir $OUTPUTDIR --config-file {target_dir}/config_stereo.yaml >$LOG 2>&1" ) f.close() @@ -205,12 +205,12 @@ def main(): print("***** Submitting processess to the cluster...") print( - "Process name: " + target_dir.split("/")[-2:][1] + "_stereo" + str(nsblvl) + "Process name: " + target_dir.split("/")[-2:][1] + "_stereo_" + str(nsblvl) ) print( "To check the jobs submitted to the cluster, type: squeue -n " + target_dir.split("/")[-2:][1] - + "_stereo" + + "_stereo_" + str(nsblvl) )