Minor fixes (comments...)

nsb_scripts/LSTnsb.py

@@ -1,3 +1,6 @@
+"""
+Evaluates NSB level for a LST run
+"""
 import os
 import argparse
 from lstchain.image.modifier import calculate_noise_parameters
@@ -13,9 +16,7 @@
 
 
 def main():
-    """
-    create list of LST runs with nsb
-    """
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--config-file",

nsb_scripts/collect_nsb.py

@@ -1,3 +1,6 @@
+"""
+Loop on the files produced by the parallel jobs to create a list of LST runs for each NSB level
+"""
 import yaml
 import argparse
 import glob

nsb_scripts/config_general.yaml

@@ -7,7 +7,8 @@ mc_tel_ids:
     MAGIC-II: 3
 
 directories:
-    workspace_dir : "/fefs/aswg/workspace/elisa.visentin/MAGIC_LST_analysis/PG1553_nsb/" #Pay attention to put the last "/"!!!
+    workspace_dir : "/fefs/aswg/workspace/elisa.visentin/MAGIC_LST_analysis/PG1553_nsb" 
+    scripts_dir : "/fefs/aswg/workspace/elisa.visentin/MAGIC_LST_analysis/NSB_WIP"
     target_name   : "PG1553"
 
 general:

nsb_scripts/nsb_coincident_events.py

@@ -1,7 +1,7 @@
 """
 This scripts facilitates the usage of the script
 "lst1_magic_event_coincidence.py". This script is
-more like a "maneger" that organizes the analysis
+more like a "manager" that organizes the analysis
 process by:
 1) Creating the bash scripts for looking for
 coincidence events between MAGIC and LST in each
@@ -11,7 +11,7 @@
 
 
 Usage:
-$ python coincident_events.py
+$ python coincident_events.py (-c config_file.yaml)
 
 
 """
@@ -68,21 +68,26 @@ def configfile_coincidence(ids, target_dir):
 
 def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date):
     """
-    This function links the LST data paths to the working directory. This is a preparation step required for running lst1_magic_event_coincidence.py
-
+    This function links the LST data paths to the working directory and creates bash scripts. 
     Parameters
     ----------
+    scripts_dir: str
+        Path to the scripts directory
     target_dir: str
         Path to the working directory
     LST_runs: matrix of strings
         This matrix is imported from config_general.yaml and tells the function where to find the LST data and link them to our working directory
+    nsb: int
+        NSB level
+    date:
+        Array of lists [date run] for all the LST runs (no NSB splitting) 
     """
 
-    coincidence_DL1_dir = target_dir + f"/v_{__version__}"
-    if not os.path.exists(coincidence_DL1_dir + "/DL1_Coincident/"):
-        os.mkdir(f"{coincidence_DL1_dir}/DL1_Coincident")
+    coincidence_DL1_dir = target_dir + f"/v{__version__}"
+    if not os.path.exists(coincidence_DL1_dir + "/DL1Coincident/"):
+        os.mkdir(f"{coincidence_DL1_dir}/DL1Coincident")
     ST_list = [
-        os.path.basename(x) for x in glob.glob(f"{target_dir}/v_{__version__}/DL1/*")
+        os.path.basename(x) for x in glob.glob(f"{target_dir}/v{__version__}/DL1/*")
     ]
 
     if (len(LST_runs) == 2) and (len(LST_runs[0]) == 10):
@@ -97,9 +102,9 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date):
         date.append(dt)
 
     for p in ST_list:
-        MAGIC_DL1_dir = target_dir + f"/v_{__version__}" + "/DL1/" + p
-        if not os.path.exists(coincidence_DL1_dir + "/DL1_Coincident/" + str(p)):
-            os.mkdir(f"{coincidence_DL1_dir}/DL1_Coincident/{p}")
+        MAGIC_DL1_dir = target_dir + f"/v{__version__}" + "/DL1/" + p
+        if not os.path.exists(coincidence_DL1_dir + "/DL1Coincident/" + str(p)):
+            os.mkdir(f"{coincidence_DL1_dir}/DL1Coincident/{p}")
         dates = [os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M1/*")]
         for d in dates:
             Y_M = int(d.split("_")[0])
@@ -117,13 +122,13 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date):
                 if day_MAGIC == day_LST + delta:
                     if not os.path.exists(
                         coincidence_DL1_dir
-                        + "/DL1_Coincident/"
+                        + "/DL1Coincident/"
                         + str(p)
-                        + "/"
+                        + "/NSB"
                         + str(nsb)
                     ):
                         os.mkdir(
-                            f"{coincidence_DL1_dir}/DL1_Coincident/{p}" + "/" + str(nsb)
+                            f"{coincidence_DL1_dir}/DL1Coincident/{p}" + "/NSB" + str(nsb)
                         )
 
                     lstObsDir = (
@@ -133,67 +138,67 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date):
                     inputdir = f"/fefs/aswg/data/real/DL1/{lstObsDir}/v0.9/tailcut84"
                     if not os.path.exists(
                         coincidence_DL1_dir
-                        + "/DL1_Coincident/"
+                        + "/DL1Coincident/"
                         + str(p)
-                        + "/"
+                        + "/NSB"
                         + str(nsb)
                         + "/"
                         + lstObsDir
                     ):
                         os.mkdir(
-                            f"{coincidence_DL1_dir}/DL1_Coincident/{p}"
-                            + "/"
+                            f"{coincidence_DL1_dir}/DL1Coincident/{p}"
+                            + "/NSB"
                             + str(nsb)
                             + "/"
                             + lstObsDir
                         )
                     if not os.path.exists(
                         coincidence_DL1_dir
-                        + "/DL1_Coincident/"
+                        + "/DL1Coincident/"
                         + str(p)
-                        + "/"
+                        + "/NSB"
                         + str(nsb)
                         + "/"
                         + lstObsDir
-                        + "/log"
+                        + "/logs"
                     ):
                         os.mkdir(
-                            f"{coincidence_DL1_dir}/DL1_Coincident/{p}"
-                            + "/"
+                            f"{coincidence_DL1_dir}/DL1Coincident/{p}"
+                            + "/NSB"
                             + str(nsb)
                             + "/"
                             + lstObsDir
-                            + "/log"
+                            + "/logs"
                         )
 
                     outputdir = (
-                        f"{coincidence_DL1_dir}/DL1_Coincident/{p}"
-                        + "/"
+                        f"{coincidence_DL1_dir}/DL1Coincident/{p}"
+                        + "/NSB"
                         + str(nsb)
                         + "/"
                         + lstObsDir
                     )
                     list_of_subruns = np.sort(
                         glob.glob(f"{inputdir}/dl1*Run*{i[1]}*.*.h5")
                     )
-                    if os.path.exists(f"{outputdir}/log/list_LST.txt"):
-                        with open(f"{outputdir}/log/list_LST.txt", "a") as LSTdataPathFile:
+                    if os.path.exists(f"{outputdir}/logs/list_LST.txt"):
+                        with open(f"{outputdir}/logs/list_LST.txt", "a") as LSTdataPathFile:
                             for subrun in list_of_subruns:
                                 LSTdataPathFile.write(
                                     subrun + "\n"
                                 )  # If this files already exists, simply append the new information
                     else:
                         f = open(
-                            f"{outputdir}/log/list_LST.txt", "w"
+                            f"{outputdir}/logs/list_LST.txt", "w"
                         )  # If the file list_LST.txt does not exist, it will be created here
                         for subrun in list_of_subruns:
                             f.write(subrun + "\n")
                         f.close()
 
-                    if not os.path.exists(outputdir + "/log/list_LST.txt"):
+                    if not os.path.exists(outputdir + "/logs/list_LST.txt"):
                         continue
                     process_size = (
-                        len(np.genfromtxt(outputdir + "/log/list_LST.txt", dtype="str")) - 1
+                        len(np.genfromtxt(outputdir + "/logs/list_LST.txt", dtype="str")) - 1
                     )
 
                     if process_size < 0:
@@ -219,10 +224,10 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date):
                         f"export INM={MAGIC_DL1_dir}/Merged/Merged_{str(Y_M).zfill(4)}_{str(M_M).zfill(2)}_{str(D_M).zfill(2)}\n"
                     )
                     f.write(f"export OUTPUTDIR={outputdir}\n")
-                    f.write("SAMPLE_LIST=($(<$OUTPUTDIR/log/list_LST.txt))\n")
+                    f.write("SAMPLE_LIST=($(<$OUTPUTDIR/logs/list_LST.txt))\n")
                     f.write("SAMPLE=${SAMPLE_LIST[${SLURM_ARRAY_TASK_ID}]}\n")
                     f.write(
-                        "export LOG=$OUTPUTDIR/log/coincidence_${SLURM_ARRAY_TASK_ID}.log\n"
+                        "export LOG=$OUTPUTDIR/logs/coincidence_${SLURM_ARRAY_TASK_ID}.log\n"
                     )
                     f.write(
                         f"conda run -n magic-lst python {scripts_dir}/lst1_magic_event_coincidence.py --input-file-lst $SAMPLE --input-dir-magic $INM --output-dir $OUTPUTDIR --config-file {target_dir}/config_coincidence.yaml >$LOG 2>&1"
@@ -232,7 +237,7 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date):
 
 def main():
     """
-    Here we read the config_general.yaml file and call the functions defined above.
+    Here we read the config file and call the functions defined above.
     """
 
     parser = argparse.ArgumentParser()

nsb_scripts/nsb_level.py

@@ -1,3 +1,7 @@
+"""
+Bash scripts to run LSTnsb.py on all the LST runs by using parallel jobs
+"""
+
 import argparse
 import os
 import numpy as np

nsb_scripts/nsb_merge_M1_M2_night.py

@@ -1,23 +1,11 @@
 """
-This script split the proton MC data sample into "train"
-and "test", deletes possible failed runs (only those files
-that end up with a size < 1 kB), and generates the bash 
-scripts to merge the data files calling the script "merge_hdf_files.py"
-in the follwoing order:
-
-MAGIC:
-1) Merge the subruns into runs for M1 and M2 individually.
-2) Merge the runs of M1 and M2 into M1-M2 runs.
-3) Merge all the M1-M2 runs for a given night.
-Workingdir/DL1/Observations/Merged 
-
-MC:
-1) Merges all MC runs in a node and save them at
-Workingdir/DL1/MC/PARTICLE/Merged 
-
+This script generates the bash scripts to merge the data 
+files calling the script "merge_hdf_files.py" to 
+merge all the M1-M2 runs for a given night.
 
 Usage:
-$ python merging_runs_and_spliting_training_samples.py
+$ python merge_M1_M2_night.py 
+(-c config_file.yaml)
 
 """
 import argparse
@@ -36,25 +24,23 @@
 
 def merge(scripts_dir, target_dir):
     """
-    This function creates the bash scripts to run merge_hdf_files.py in all MAGIC subruns.
+    This function creates the bash scripts to run merge_hdf_files.py 
 
     Parameters
     ----------
+    scripts_dir: str
+        Path to the scripts directory
     target_dir: str
         Path to the working directory
-    identification: str
-        Tells which batch to create. Options: subruns, M1M2, nights
-    MAGIC_runs: matrix of strings
-        This matrix is imported from config_general.yaml and tells the function where to find the data and where to put the merged files
-    """
+     """
     ST_list = [
-        os.path.basename(x) for x in glob.glob(f"{target_dir}/v_{__version__}/DL1/*")
+        os.path.basename(x) for x in glob.glob(f"{target_dir}/v{__version__}/DL1/*")
     ]
 
     for p in ST_list:
         process_name = "merging_" + target_dir.split("/")[-2:][1]
 
-        MAGIC_DL1_dir = target_dir + f"/v_{__version__}" + "/DL1/" + p
+        MAGIC_DL1_dir = target_dir + f"/v{__version__}" + "/DL1/" + p
         if os.path.exists(MAGIC_DL1_dir + "/M1") & os.path.exists(
             MAGIC_DL1_dir + "/M2"
         ):
@@ -77,18 +63,18 @@ def merge(scripts_dir, target_dir):
                 os.mkdir(
                     f"{MAGIC_DL1_dir}/Merged/Merged_{i}"
                 )  # Creating a merged directory for each night
-            if not os.path.exists(MAGIC_DL1_dir + f"/Merged/Merged_{i}/log"):
-                os.mkdir(f"{MAGIC_DL1_dir}/Merged/Merged_{i}/log")
+            if not os.path.exists(MAGIC_DL1_dir + f"/Merged/Merged_{i}/logs"):
+                os.mkdir(f"{MAGIC_DL1_dir}/Merged/Merged_{i}/logs")
             f.write(
-                f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --output-dir {MAGIC_DL1_dir}/Merged/Merged_{i} >{MAGIC_DL1_dir}/Merged/Merged_{i}/log/merge_night_{i}.log \n"
+                f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --output-dir {MAGIC_DL1_dir}/Merged/Merged_{i} >{MAGIC_DL1_dir}/Merged/Merged_{i}/logs/merge_night_{i}.log \n"
             )
 
         f.close()
 
 
 def main():
     """
-    Here we read the config_general.yaml file, split the pronton sample into "test" and "train", and merge the MAGIC files.
+    Here we read the config_general.yaml file and merge the MAGIC files.
     """
 
     parser = argparse.ArgumentParser()
@@ -116,7 +102,7 @@ def main():
     print("***** Generating merge bashscripts...")
     merge(
         scripts_dir, target_dir
-    )  # generating the bash script to merge the subruns
+    ) 
 
     print("***** Running merge_hdf_files.py in the MAGIC data files...")
     print("Process name: merging_" + target_dir.split("/")[-2:][1])