Pyflakes + Path (user directories) + no overwrite DL1 MAGIC

nsb_scripts/PG1553_config_general.yaml

@@ -7,8 +7,8 @@ mc_tel_ids:
     MAGIC-II: 3
 
 directories:
-    workspace_dir : "/fefs/aswg/workspace/elisa.visentin/" #Pay attention to put the last "/"!!!
-    scripts_dir : "/fefs/aswg/workspace/elisa.visentin/MAGIC_LST_analysis/NSB_WIP/"
+    workspace_dir : "/fefs/aswg/workspace/elisa.visentin" 
+    scripts_dir : "/fefs/aswg/workspace/elisa.visentin/MAGIC_LST_analysis/NSB_WIP"
     target_name   : PG1553
 
 

nsb_scripts/nsb_DL1_to_DL2.py

@@ -15,7 +15,7 @@
 import glob
 import yaml
 import logging
-
+from pathlib import Path
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler())
 logger.setLevel(logging.INFO)
@@ -98,8 +98,8 @@ def main():
 
 
 
-    target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"]
-    scripts_dir=config["directories"]["scripts_dir"]
+    target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"])
+    scripts_dir=str(Path(config["directories"]["scripts_dir"]))
     source=config['directories']['target_name']
     listnsb = np.sort(glob.glob(f"{source}_LST_*_.txt"))
     nsb=[]

nsb_scripts/nsb_DL2_to_DL3.py

@@ -110,8 +110,8 @@ def main():
 
     telescope_ids = list(config["mc_tel_ids"].values())
 
-    target_dir = str(Path(config["directories"]["workspace_dir"]))+"/"+config["directories"]["target_name"]
-    scripts_dir=config["directories"]["scripts_dir"]
+    target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"])
+    scripts_dir=str(Path(config["directories"]["scripts_dir"]))
 
 
     target_coords = [config["general"]["target_RA_deg"],config["general"]["target_Dec_deg"]]

nsb_scripts/nsb_coincident_events.py

@@ -13,7 +13,9 @@
 Usage:
 $ python coincident_events.py
 
+
 """
+from pathlib import Path
 import argparse
 import os
 import numpy as np
@@ -178,8 +180,8 @@ def main():
         config = yaml.safe_load(f)
 
     telescope_ids = list(config["mc_tel_ids"].values())
-    target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"]
-    scripts_dir=config["directories"]["scripts_dir"]
+    target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"])
+    scripts_dir=str(Path(config["directories"]["scripts_dir"]))
     source=config['directories']['target_name']
     print("***** Generating file config_coincidence.yaml...")
     print("***** This file can be found in ",target_dir)

nsb_scripts/nsb_level.py

@@ -50,7 +50,7 @@ def main():
         print(i)
         i=i.rstrip()
         bash_scripts(i,args.config_file)
-    list_of_bash_scripts = np.sort(glob.glob(f"run_*.sh"))
+    list_of_bash_scripts = np.sort(glob.glob("run_*.sh"))
     print(list_of_bash_scripts)   
     for n,run in enumerate(list_of_bash_scripts):
         if n == 0:

nsb_scripts/nsb_merge_DL2.py

@@ -82,9 +82,9 @@ def main():
 
 
 
-    target_dir = str(Path(config["directories"]["workspace_dir"]))+"/"+config["directories"]["target_name"]
+    target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"])
 
-    scripts_dir=config["directories"]["scripts_dir"]
+    scripts_dir=str(Path(config["directories"]["scripts_dir"]))
     source=config['directories']['target_name']
 
     listnsb = np.sort(glob.glob(f"{source}_LST_*_.txt"))
@@ -98,7 +98,7 @@ def main():
       MergeDL2(scripts_dir, target_dir, nsblvl)
 
     #Below we run the bash scripts to perform the DL1 to DL2 cnoversion:
-    list_of_DL1_to_2_scripts = np.sort(glob.glob("DL3_*.sh"))
+    list_of_DL1_to_2_scripts = np.sort(glob.glob("DL3_*_0_*.sh"))
     print(list_of_DL1_to_2_scripts)
     for n,run in enumerate(list_of_DL1_to_2_scripts):
         if n == 0:

nsb_scripts/nsb_merging_runs_and_spliting_training_samples.py

@@ -26,7 +26,7 @@
 import glob
 import yaml
 import logging
-
+from pathlib import Path
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler())
@@ -119,8 +119,8 @@ def main():
         config = yaml.safe_load(f)
 
 
-    target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"]
-    scripts_dir=config["directories"]["scripts_dir"]
+    target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"])
+    scripts_dir=str(Path(config["directories"]["scripts_dir"]))
 
     MAGIC_runs_and_dates = config["general"]["MAGIC_runs"]
     MAGIC_runs = np.genfromtxt(MAGIC_runs_and_dates,dtype=str,delimiter=',')

nsb_scripts/nsb_setting_up_config_and_dir.py

@@ -32,7 +32,7 @@
 import glob
 
 import yaml
-
+from pathlib import Path
 def config_file_gen(ids, target_dir):
 
     """
@@ -163,19 +163,12 @@ def directories_generator(target_dir, telescope_ids,MAGIC_runs):
 
     if not os.path.exists(target_dir):
         os.mkdir(target_dir)
+    if not os.path.exists(target_dir+"/DL1"):
         os.mkdir(target_dir+"/DL1")
+    if not os.path.exists(target_dir+"/DL1/Observations"):
         os.mkdir(target_dir+"/DL1/Observations")
 
-    else:
-        overwrite = input("Directory "+target_dir.split("/")[-1]+" already exists. Would you like to overwrite it? [only 'y' or 'n']: ")
-        if overwrite == "y":
-            os.system("rm -r "+target_dir)
-            os.mkdir(target_dir)
-            os.mkdir(target_dir+"/DL1")
-            os.mkdir(target_dir+"/DL1/Observations")
-
-        else:
-            print("Directory not modified.")
+
 
 
 
@@ -188,25 +181,33 @@ def directories_generator(target_dir, telescope_ids,MAGIC_runs):
 
         MAGIC_runs=[]
         MAGIC_runs.append(MAGIC)
-    if telescope_ids[-1] > 0:    
+    print(MAGIC_runs)
+    if telescope_ids[-1] > 0:
+        print('M2')
         if not os.path.exists(target_dir+"/DL1/Observations/M2"):
             os.mkdir(target_dir+"/DL1/Observations/M2")
-            for i in MAGIC_runs:
-                if not os.path.exists(target_dir+"/DL1/Observations/M2/"+i[0]):
-                    os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0])
-                    os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0]+"/"+i[1])
-                else:
-                    os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0]+"/"+i[1])
+        for i in MAGIC_runs:
+            print('date',i[0])
+
+            if not os.path.exists(target_dir+"/DL1/Observations/M2/"+i[0]):
+                os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0])
+                print('directory created')
+            if not os.path.exists(target_dir+"/DL1/Observations/M2/"+i[0]+"/"+i[1]):
+
+                os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0]+"/"+i[1])
+
+
 
     if telescope_ids[-2] > 0:
         if not os.path.exists(target_dir+"/DL1/Observations/M1"):
             os.mkdir(target_dir+"/DL1/Observations/M1")
-            for i in MAGIC_runs:
-                if not os.path.exists(target_dir+"/DL1/Observations/M1/"+i[0]):
-                    os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0])
-                    os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0]+"/"+i[1])
-                else:
-                    os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0]+"/"+i[1])
+        for i in MAGIC_runs:
+            if not os.path.exists(target_dir+"/DL1/Observations/M1/"+i[0]):
+                os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0])
+            if not os.path.exists(target_dir+"/DL1/Observations/M1/"+i[0]+"/"+i[1]):
+                os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0]+"/"+i[1])
+
+
 
 
 
@@ -240,8 +241,8 @@ def main():
 
     MAGIC_runs_and_dates = config["general"]["MAGIC_runs"]
     MAGIC_runs = np.genfromtxt(MAGIC_runs_and_dates,dtype=str,delimiter=',') #READ LIST OF DATES AND RUNS: format table where each line is like "2020_11_19,5093174"
-    target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"]
-    scripts_dir=config["directories"]["scripts_dir"]
+    target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"])
+    scripts_dir=str(Path(config["directories"]["scripts_dir"]))
 
 
 

nsb_scripts/nsb_stereo_events.py

@@ -13,7 +13,7 @@
 import glob
 import yaml
 import logging
-
+from pathlib import Path
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler())
 logger.setLevel(logging.INFO)
@@ -114,8 +114,8 @@ def main():
 
 
 
-    target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"]
-    scripts_dir=config["directories"]["scripts_dir"]
+    target_dir = str(Path(config["directories"]["workspace_dir"])  / config["directories"]["target_name"])
+    scripts_dir=str(Path(config["directories"]["scripts_dir"]))
     telescope_ids = list(config["mc_tel_ids"].values())
     source=config['directories']['target_name']
     print("***** Generating file config_stereo.yaml...")

nsb_scripts/spreadsheets/GoogleSheets_to_h5.py

@@ -1,6 +1,6 @@
 import pandas as pd
-import os
-import pickle
+
+
 import json
 from google.oauth2 import service_account
 import gspread