From f1e06a4d38aa4273e4a4c0c3be8defaa5ba10052 Mon Sep 17 00:00:00 2001 From: Elisa-Visentin Date: Fri, 8 Sep 2023 08:57:53 +0000 Subject: [PATCH] Pyflakes + Path (user directories) + no overwrite DL1 MAGIC --- nsb_scripts/PG1553_config_general.yaml | 4 +- nsb_scripts/nsb_DL1_to_DL2.py | 6 +-- nsb_scripts/nsb_DL2_to_DL3.py | 4 +- nsb_scripts/nsb_coincident_events.py | 6 ++- nsb_scripts/nsb_level.py | 2 +- nsb_scripts/nsb_merge_DL2.py | 6 +-- ...ging_runs_and_spliting_training_samples.py | 6 +-- nsb_scripts/nsb_setting_up_config_and_dir.py | 53 ++++++++++--------- nsb_scripts/nsb_stereo_events.py | 6 +-- .../spreadsheets/GoogleSheets_to_h5.py | 4 +- 10 files changed, 50 insertions(+), 47 deletions(-) diff --git a/nsb_scripts/PG1553_config_general.yaml b/nsb_scripts/PG1553_config_general.yaml index 142a51b4f..27e838e75 100644 --- a/nsb_scripts/PG1553_config_general.yaml +++ b/nsb_scripts/PG1553_config_general.yaml @@ -7,8 +7,8 @@ mc_tel_ids: MAGIC-II: 3 directories: - workspace_dir : "/fefs/aswg/workspace/elisa.visentin/" #Pay attention to put the last "/"!!! - scripts_dir : "/fefs/aswg/workspace/elisa.visentin/MAGIC_LST_analysis/NSB_WIP/" + workspace_dir : "/fefs/aswg/workspace/elisa.visentin" + scripts_dir : "/fefs/aswg/workspace/elisa.visentin/MAGIC_LST_analysis/NSB_WIP" target_name : PG1553 diff --git a/nsb_scripts/nsb_DL1_to_DL2.py b/nsb_scripts/nsb_DL1_to_DL2.py index 288f73e17..649c1b1e4 100644 --- a/nsb_scripts/nsb_DL1_to_DL2.py +++ b/nsb_scripts/nsb_DL1_to_DL2.py @@ -15,7 +15,7 @@ import glob import yaml import logging - +from pathlib import Path logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) @@ -98,8 +98,8 @@ def main(): - target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"] - scripts_dir=config["directories"]["scripts_dir"] + target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"]) + scripts_dir=str(Path(config["directories"]["scripts_dir"])) source=config['directories']['target_name'] listnsb = np.sort(glob.glob(f"{source}_LST_*_.txt")) nsb=[] diff --git a/nsb_scripts/nsb_DL2_to_DL3.py b/nsb_scripts/nsb_DL2_to_DL3.py index 6452db712..2ebc4bcd1 100644 --- a/nsb_scripts/nsb_DL2_to_DL3.py +++ b/nsb_scripts/nsb_DL2_to_DL3.py @@ -110,8 +110,8 @@ def main(): telescope_ids = list(config["mc_tel_ids"].values()) - target_dir = str(Path(config["directories"]["workspace_dir"]))+"/"+config["directories"]["target_name"] - scripts_dir=config["directories"]["scripts_dir"] + target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"]) + scripts_dir=str(Path(config["directories"]["scripts_dir"])) target_coords = [config["general"]["target_RA_deg"],config["general"]["target_Dec_deg"]] diff --git a/nsb_scripts/nsb_coincident_events.py b/nsb_scripts/nsb_coincident_events.py index e80d3521b..3bbc0e4fb 100644 --- a/nsb_scripts/nsb_coincident_events.py +++ b/nsb_scripts/nsb_coincident_events.py @@ -13,7 +13,9 @@ Usage: $ python coincident_events.py + """ +from pathlib import Path import argparse import os import numpy as np @@ -178,8 +180,8 @@ def main(): config = yaml.safe_load(f) telescope_ids = list(config["mc_tel_ids"].values()) - target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"] - scripts_dir=config["directories"]["scripts_dir"] + target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"]) + scripts_dir=str(Path(config["directories"]["scripts_dir"])) source=config['directories']['target_name'] print("***** Generating file config_coincidence.yaml...") print("***** This file can be found in ",target_dir) diff --git a/nsb_scripts/nsb_level.py b/nsb_scripts/nsb_level.py index 2909e0578..cbeec36c7 100644 --- a/nsb_scripts/nsb_level.py +++ b/nsb_scripts/nsb_level.py @@ -50,7 +50,7 @@ def main(): print(i) i=i.rstrip() bash_scripts(i,args.config_file) - list_of_bash_scripts = np.sort(glob.glob(f"run_*.sh")) + list_of_bash_scripts = np.sort(glob.glob("run_*.sh")) print(list_of_bash_scripts) for n,run in enumerate(list_of_bash_scripts): if n == 0: diff --git a/nsb_scripts/nsb_merge_DL2.py b/nsb_scripts/nsb_merge_DL2.py index 05807505f..1fa0c1c06 100644 --- a/nsb_scripts/nsb_merge_DL2.py +++ b/nsb_scripts/nsb_merge_DL2.py @@ -82,9 +82,9 @@ def main(): - target_dir = str(Path(config["directories"]["workspace_dir"]))+"/"+config["directories"]["target_name"] + target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"]) - scripts_dir=config["directories"]["scripts_dir"] + scripts_dir=str(Path(config["directories"]["scripts_dir"])) source=config['directories']['target_name'] listnsb = np.sort(glob.glob(f"{source}_LST_*_.txt")) @@ -98,7 +98,7 @@ def main(): MergeDL2(scripts_dir, target_dir, nsblvl) #Below we run the bash scripts to perform the DL1 to DL2 cnoversion: - list_of_DL1_to_2_scripts = np.sort(glob.glob("DL3_*.sh")) + list_of_DL1_to_2_scripts = np.sort(glob.glob("DL3_*_0_*.sh")) print(list_of_DL1_to_2_scripts) for n,run in enumerate(list_of_DL1_to_2_scripts): if n == 0: diff --git a/nsb_scripts/nsb_merging_runs_and_spliting_training_samples.py b/nsb_scripts/nsb_merging_runs_and_spliting_training_samples.py index 72dc4a932..2e66e7300 100644 --- a/nsb_scripts/nsb_merging_runs_and_spliting_training_samples.py +++ b/nsb_scripts/nsb_merging_runs_and_spliting_training_samples.py @@ -26,7 +26,7 @@ import glob import yaml import logging - +from pathlib import Path logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) @@ -119,8 +119,8 @@ def main(): config = yaml.safe_load(f) - target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"] - scripts_dir=config["directories"]["scripts_dir"] + target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"]) + scripts_dir=str(Path(config["directories"]["scripts_dir"])) MAGIC_runs_and_dates = config["general"]["MAGIC_runs"] MAGIC_runs = np.genfromtxt(MAGIC_runs_and_dates,dtype=str,delimiter=',') diff --git a/nsb_scripts/nsb_setting_up_config_and_dir.py b/nsb_scripts/nsb_setting_up_config_and_dir.py index 0aa88e0bb..9e4f675f5 100644 --- a/nsb_scripts/nsb_setting_up_config_and_dir.py +++ b/nsb_scripts/nsb_setting_up_config_and_dir.py @@ -32,7 +32,7 @@ import glob import yaml - +from pathlib import Path def config_file_gen(ids, target_dir): """ @@ -163,19 +163,12 @@ def directories_generator(target_dir, telescope_ids,MAGIC_runs): if not os.path.exists(target_dir): os.mkdir(target_dir) + if not os.path.exists(target_dir+"/DL1"): os.mkdir(target_dir+"/DL1") + if not os.path.exists(target_dir+"/DL1/Observations"): os.mkdir(target_dir+"/DL1/Observations") - else: - overwrite = input("Directory "+target_dir.split("/")[-1]+" already exists. Would you like to overwrite it? [only 'y' or 'n']: ") - if overwrite == "y": - os.system("rm -r "+target_dir) - os.mkdir(target_dir) - os.mkdir(target_dir+"/DL1") - os.mkdir(target_dir+"/DL1/Observations") - - else: - print("Directory not modified.") + @@ -188,25 +181,33 @@ def directories_generator(target_dir, telescope_ids,MAGIC_runs): MAGIC_runs=[] MAGIC_runs.append(MAGIC) - if telescope_ids[-1] > 0: + print(MAGIC_runs) + if telescope_ids[-1] > 0: + print('M2') if not os.path.exists(target_dir+"/DL1/Observations/M2"): os.mkdir(target_dir+"/DL1/Observations/M2") - for i in MAGIC_runs: - if not os.path.exists(target_dir+"/DL1/Observations/M2/"+i[0]): - os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0]) - os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0]+"/"+i[1]) - else: - os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0]+"/"+i[1]) + for i in MAGIC_runs: + print('date',i[0]) + + if not os.path.exists(target_dir+"/DL1/Observations/M2/"+i[0]): + os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0]) + print('directory created') + if not os.path.exists(target_dir+"/DL1/Observations/M2/"+i[0]+"/"+i[1]): + + os.mkdir(target_dir+"/DL1/Observations/M2/"+i[0]+"/"+i[1]) + + if telescope_ids[-2] > 0: if not os.path.exists(target_dir+"/DL1/Observations/M1"): os.mkdir(target_dir+"/DL1/Observations/M1") - for i in MAGIC_runs: - if not os.path.exists(target_dir+"/DL1/Observations/M1/"+i[0]): - os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0]) - os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0]+"/"+i[1]) - else: - os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0]+"/"+i[1]) + for i in MAGIC_runs: + if not os.path.exists(target_dir+"/DL1/Observations/M1/"+i[0]): + os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0]) + if not os.path.exists(target_dir+"/DL1/Observations/M1/"+i[0]+"/"+i[1]): + os.mkdir(target_dir+"/DL1/Observations/M1/"+i[0]+"/"+i[1]) + + @@ -240,8 +241,8 @@ def main(): MAGIC_runs_and_dates = config["general"]["MAGIC_runs"] MAGIC_runs = np.genfromtxt(MAGIC_runs_and_dates,dtype=str,delimiter=',') #READ LIST OF DATES AND RUNS: format table where each line is like "2020_11_19,5093174" - target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"] - scripts_dir=config["directories"]["scripts_dir"] + target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"]) + scripts_dir=str(Path(config["directories"]["scripts_dir"])) diff --git a/nsb_scripts/nsb_stereo_events.py b/nsb_scripts/nsb_stereo_events.py index 32d6b4aeb..a29e34ccc 100644 --- a/nsb_scripts/nsb_stereo_events.py +++ b/nsb_scripts/nsb_stereo_events.py @@ -13,7 +13,7 @@ import glob import yaml import logging - +from pathlib import Path logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.INFO) @@ -114,8 +114,8 @@ def main(): - target_dir = config["directories"]["workspace_dir"]+config["directories"]["target_name"] - scripts_dir=config["directories"]["scripts_dir"] + target_dir = str(Path(config["directories"]["workspace_dir"]) / config["directories"]["target_name"]) + scripts_dir=str(Path(config["directories"]["scripts_dir"])) telescope_ids = list(config["mc_tel_ids"].values()) source=config['directories']['target_name'] print("***** Generating file config_stereo.yaml...") diff --git a/nsb_scripts/spreadsheets/GoogleSheets_to_h5.py b/nsb_scripts/spreadsheets/GoogleSheets_to_h5.py index 595bd1281..3132e51c9 100644 --- a/nsb_scripts/spreadsheets/GoogleSheets_to_h5.py +++ b/nsb_scripts/spreadsheets/GoogleSheets_to_h5.py @@ -1,6 +1,6 @@ import pandas as pd -import os -import pickle + + import json from google.oauth2 import service_account import gspread