black

cta-observatory · Sep 19, 2023 · 29e0bf1 · 29e0bf1
1 parent 5ddc02f
commit 29e0bf1
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 77 deletions.
diff --git a/nsb_scripts/LSTnsb.py b/nsb_scripts/LSTnsb.py
@@ -16,7 +16,6 @@
 
 
 def main():
-
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--config-file",

diff --git a/nsb_scripts/nsb_coincident_events.py b/nsb_scripts/nsb_coincident_events.py
@@ -68,7 +68,7 @@ def configfile_coincidence(ids, target_dir):
 
 def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date, source):
     """
-    This function links the LST data paths to the working directory and creates bash scripts. 
+    This function links the LST data paths to the working directory and creates bash scripts.
     Parameters
     ----------
     scripts_dir: str
@@ -80,7 +80,7 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date, source):
     nsb: int
         NSB level
     date:
-        Array of lists [date run] for all the LST runs (no NSB splitting) 
+        Array of lists [date run] for all the LST runs (no NSB splitting)
     """
 
     coincidence_DL1_dir = target_dir + f"/v{__version__}"
@@ -128,7 +128,9 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date, source):
                         + str(nsb)
                     ):
                         os.mkdir(
-                            f"{coincidence_DL1_dir}/DL1Coincident/{p}" + "/NSB" + str(nsb)
+                            f"{coincidence_DL1_dir}/DL1Coincident/{p}"
+                            + "/NSB"
+                            + str(nsb)
                         )
 
                     lstObsDir = (
@@ -182,7 +184,9 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date, source):
                         glob.glob(f"{inputdir}/dl1*Run*{i[1]}*.*.h5")
                     )
                     if os.path.exists(f"{outputdir}/logs/list_LST.txt"):
-                        with open(f"{outputdir}/logs/list_LST.txt", "a") as LSTdataPathFile:
+                        with open(
+                            f"{outputdir}/logs/list_LST.txt", "a"
+                        ) as LSTdataPathFile:
                             for subrun in list_of_subruns:
                                 LSTdataPathFile.write(
                                     subrun + "\n"
@@ -198,12 +202,18 @@ def linking_bash_lst(scripts_dir, target_dir, LST_runs, nsb, date, source):
                     if not os.path.exists(outputdir + "/logs/list_LST.txt"):
                         continue
                     process_size = (
-                        len(np.genfromtxt(outputdir + "/logs/list_LST.txt", dtype="str")) - 1
+                        len(
+                            np.genfromtxt(outputdir + "/logs/list_LST.txt", dtype="str")
+                        )
+                        - 1
                     )
 
                     if process_size < 0:
                         continue
-                    f = open(f"{source}_LST_coincident_{nsb}_{outputdir.split('/')[-1]}.sh", "w")
+                    f = open(
+                        f"{source}_LST_coincident_{nsb}_{outputdir.split('/')[-1]}.sh",
+                        "w",
+                    )
                     f.write("#!/bin/sh\n\n")
                     f.write("#SBATCH -p short\n")
                     f.write(

diff --git a/nsb_scripts/nsb_level.py b/nsb_scripts/nsb_level.py
@@ -15,7 +15,6 @@
 
 
 def bash_scripts(run, config, source):
-
     f = open(f"{source}_run_{run}.sh", "w")
     f.write("#!/bin/sh\n\n")
     f.write("#SBATCH -p long\n")
@@ -48,7 +47,7 @@ def main():
         args.config_file, "rb"
     ) as f:  # "rb" mode opens the file in binary format for reading
         config = yaml.safe_load(f)
-    source=config["directories"]["target_name"]
+    source = config["directories"]["target_name"]
     runs = config["general"]["LST_runs"]
 
     with open(str(runs), "r") as LSTfile:
@@ -58,9 +57,7 @@ def main():
         i = i.rstrip()
         bash_scripts(i, args.config_file, source)
     print("Process name: nsb")
-    print(
-        "To check the jobs submitted to the cluster, type: squeue -n nsb"        
-    )
+    print("To check the jobs submitted to the cluster, type: squeue -n nsb")
     list_of_bash_scripts = np.sort(glob.glob(f"{source}_run_*.sh"))
 
     if len(list_of_bash_scripts) < 1:

diff --git a/nsb_scripts/nsb_merge_M1_M2_night.py b/nsb_scripts/nsb_merge_M1_M2_night.py
@@ -24,15 +24,15 @@
 
 def merge(scripts_dir, target_dir, source):
     """
-    This function creates the bash scripts to run merge_hdf_files.py 
+    This function creates the bash scripts to run merge_hdf_files.py
 
     Parameters
     ----------
     scripts_dir: str
         Path to the scripts directory
     target_dir: str
         Path to the working directory
-     """
+    """
     ST_list = [
         os.path.basename(x) for x in glob.glob(f"{target_dir}/v{__version__}/DL1/*")
     ]
@@ -41,7 +41,6 @@ def merge(scripts_dir, target_dir, source):
         process_name = "merging_" + target_dir.split("/")[-2:][1]
 
         MAGIC_DL1_dir = target_dir + f"/v{__version__}" + "/DL1/" + p
-
 
         f = open(f"{source}_Merge_2_{p}.sh", "w")
         f.write("#!/bin/sh\n\n")
@@ -52,7 +51,6 @@ def merge(scripts_dir, target_dir, source):
         f.write("ulimit -s unlimited\n")
         f.write("ulimit -a\n\n")
 
-
         dates = [os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M1/*")]
         for i in dates:
             if not os.path.exists(MAGIC_DL1_dir + f"/Merged/Merged_{i}"):
@@ -93,14 +91,12 @@ def main():
         Path(config["directories"]["workspace_dir"])
         / config["directories"]["target_name"]
     )
-    source=config["directories"]["target_name"]
+    source = config["directories"]["target_name"]
     scripts_dir = str(Path(config["directories"]["scripts_dir"]))
 
     print("***** Generating merge bashscripts...")
-    merge(
-        scripts_dir, target_dir, source
-    ) 
-
+    merge(scripts_dir, target_dir, source)
+
     print("***** Running merge_hdf_files.py in the MAGIC data files...")
     print("Process name: merging_" + target_dir.split("/")[-2:][1])
     print(
@@ -116,10 +112,7 @@ def main():
         if n == 0:
             launch_jobs = f"merging{n}=$(sbatch --parsable {run})"
         else:
-            launch_jobs = (
-                launch_jobs
-                + f" && merging{n}=$(sbatch --parsable {run})"
-            )
+            launch_jobs = launch_jobs + f" && merging{n}=$(sbatch --parsable {run})"
 
     # print(launch_jobs)
     os.system(launch_jobs)

diff --git a/nsb_scripts/nsb_merge_M1_M2_runs.py b/nsb_scripts/nsb_merge_M1_M2_runs.py
@@ -24,7 +24,7 @@
 
 def merge(scripts_dir, target_dir, source):
     """
-    This function creates the bash scripts to run merge_hdf_files.py 
+    This function creates the bash scripts to run merge_hdf_files.py
 
     Parameters
     ----------
@@ -41,7 +41,6 @@ def merge(scripts_dir, target_dir, source):
         process_name = "merging_" + target_dir.split("/")[-2:][1]
 
         MAGIC_DL1_dir = target_dir + f"/v{__version__}" + "/DL1/" + p
-
 
         f = open(f"{source}_Merge_1_{p}.sh", "w")
         f.write("#!/bin/sh\n\n")
@@ -52,22 +51,18 @@ def merge(scripts_dir, target_dir, source):
         f.write("ulimit -s unlimited\n")
         f.write("ulimit -a\n\n")
 
-
         if os.path.exists(MAGIC_DL1_dir + "/M1") & os.path.exists(
             MAGIC_DL1_dir + "/M2"
         ):
-            dates = [
-                os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M1/*")
-            ]
+            dates = [os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M1/*")]
             for i in dates:
                 runs = [
-                    os.path.basename(x)
-                    for x in glob.glob(f"{MAGIC_DL1_dir}/M2/{i}/*")
+                    os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M2/{i}/*")
                 ]
                 for r in runs:
-
-                    if (len(os.listdir(MAGIC_DL1_dir + f"/M1/{i}/{r}"))>1) and (len(os.listdir(MAGIC_DL1_dir + f"/M2/{i}/{r}")))>1:
-                        
+                    if (len(os.listdir(MAGIC_DL1_dir + f"/M1/{i}/{r}")) > 1) and (
+                        len(os.listdir(MAGIC_DL1_dir + f"/M2/{i}/{r}"))
+                    ) > 1:
                         if not os.path.exists(MAGIC_DL1_dir + f"/Merged/{i}/Merged"):
                             os.mkdir(f"{MAGIC_DL1_dir}/Merged/{i}/Merged")
                         if not os.path.exists(
@@ -77,7 +72,6 @@ def merge(scripts_dir, target_dir, source):
                         f.write(
                             f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/Merged/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/Merged --run-wise >{MAGIC_DL1_dir}/Merged/{i}/Merged/logs/merge_{i}.log \n"
                         )
-
 
         f.close()
 
@@ -107,14 +101,12 @@ def main():
         Path(config["directories"]["workspace_dir"])
         / config["directories"]["target_name"]
     )
-    source=config["directories"]["target_name"]
+    source = config["directories"]["target_name"]
     scripts_dir = str(Path(config["directories"]["scripts_dir"]))
 
     print("***** Generating merge bashscripts...")
-    merge(
-        scripts_dir, target_dir, source
-    ) 
-
+    merge(scripts_dir, target_dir, source)
+
     print("***** Running merge_hdf_files.py in the MAGIC data files...")
     print("Process name: merging_" + target_dir.split("/")[-2:][1])
     print(
@@ -130,10 +122,7 @@ def main():
         if n == 0:
             launch_jobs = f"merging{n}=$(sbatch --parsable {run})"
         else:
-            launch_jobs = (
-                launch_jobs
-                + f" && merging{n}=$(sbatch --parsable  {run})"
-            )
+            launch_jobs = launch_jobs + f" && merging{n}=$(sbatch --parsable  {run})"
 
     # print(launch_jobs)
     os.system(launch_jobs)

diff --git a/nsb_scripts/nsb_merge_subruns.py b/nsb_scripts/nsb_merge_subruns.py
@@ -24,7 +24,7 @@
 
 def merge(scripts_dir, target_dir, source):
     """
-    This function creates the bash scripts to run merge_hdf_files.py 
+    This function creates the bash scripts to run merge_hdf_files.py
 
     Parameters
     ----------
@@ -41,7 +41,7 @@ def merge(scripts_dir, target_dir, source):
         process_name = "merging_" + target_dir.split("/")[-2:][1]
 
         MAGIC_DL1_dir = target_dir + f"/v{__version__}" + "/DL1/" + p
-        
+
         if os.path.exists(MAGIC_DL1_dir + "/M1") & os.path.exists(
             MAGIC_DL1_dir + "/M2"
         ):
@@ -56,16 +56,12 @@ def merge(scripts_dir, target_dir, source):
         f.write("ulimit -s unlimited\n")
         f.write("ulimit -a\n\n")
 
-
         if os.path.exists(MAGIC_DL1_dir + "/M1"):
-            dates = [
-                os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M1/*")
-            ]
+            dates = [os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M1/*")]
 
             for i in dates:
                 runs = [
-                    os.path.basename(x)
-                    for x in glob.glob(f"{MAGIC_DL1_dir}/M1/{i}/*")
+                    os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M1/{i}/*")
                 ]
                 if not os.path.exists(MAGIC_DL1_dir + f"/Merged/{i}"):
                     os.mkdir(
@@ -86,14 +82,11 @@ def merge(scripts_dir, target_dir, source):
                     )
 
         if os.path.exists(MAGIC_DL1_dir + "/M2"):
-            dates = [
-                os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M2/*")
-            ]
+            dates = [os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M2/*")]
 
             for i in dates:
                 runs = [
-                    os.path.basename(x)
-                    for x in glob.glob(f"{MAGIC_DL1_dir}/M2/{i}/*")
+                    os.path.basename(x) for x in glob.glob(f"{MAGIC_DL1_dir}/M2/{i}/*")
                 ]
                 if not os.path.exists(MAGIC_DL1_dir + f"/Merged/{i}"):
                     os.mkdir(
@@ -113,8 +106,6 @@ def merge(scripts_dir, target_dir, source):
                         f"conda run -n magic-lst python {scripts_dir}/merge_hdf_files.py --input-dir {MAGIC_DL1_dir}/M2/{i}/{r} --output-dir {MAGIC_DL1_dir}/Merged/{i}/{r} >{MAGIC_DL1_dir}/Merged/{i}/{r}/logs/merge_M2_{i}_{r}.log \n"
                     )
 
-
-
         f.close()
 
 
@@ -143,14 +134,14 @@ def main():
         Path(config["directories"]["workspace_dir"])
         / config["directories"]["target_name"]
     )
-    source=config["directories"]["target_name"]
+    source = config["directories"]["target_name"]
     scripts_dir = str(Path(config["directories"]["scripts_dir"]))
 
     print("***** Generating merge bashscripts...")
     merge(
         scripts_dir, target_dir, source
     )  # generating the bash script to merge the subruns
-   
+
     print("***** Running merge_hdf_files.py in the MAGIC data files...")
     print("Process name: merging_" + target_dir.split("/")[-2:][1])
     print(
@@ -166,14 +157,11 @@ def main():
         if n == 0:
             launch_jobs = f"merging{n}=$(sbatch --parsable {run})"
         else:
-            launch_jobs = (
-                launch_jobs
-                + f" && merging{n}=$(sbatch --parsable {run})"
-            )
+            launch_jobs = launch_jobs + f" && merging{n}=$(sbatch --parsable {run})"
 
     # print(launch_jobs)
     os.system(launch_jobs)
 
 
 if __name__ == "__main__":
-    main()
+    main()
diff --git a/nsb_scripts/nsb_setting_up_config_and_dir.py b/nsb_scripts/nsb_setting_up_config_and_dir.py
@@ -91,7 +91,9 @@ def config_file_gen(ids, target_dir):
     f.close()
 
 
-def lists_and_bash_gen_MAGIC(scripts_dir, target_dir, telescope_ids, MAGIC_runs, source):
+def lists_and_bash_gen_MAGIC(
+    scripts_dir, target_dir, telescope_ids, MAGIC_runs, source
+):
     """
     Below we create a bash script that links the MAGIC data paths to each subdirectory.
     """
@@ -290,8 +292,6 @@ def directories_generator(target_dir, telescope_ids, MAGIC_runs):
     Here we create all subdirectories for a given workspace and target name.
     """
 
-
-
     if not os.path.exists(target_dir):
         os.mkdir(target_dir)
     if not os.path.exists(target_dir + f"/v{__version__}"):
@@ -383,7 +383,7 @@ def main():
         Path(config["directories"]["workspace_dir"])
         / config["directories"]["target_name"]
     )
-    source=config["directories"]["target_name"]
+    source = config["directories"]["target_name"]
     scripts_dir = str(Path(config["directories"]["scripts_dir"]))
 
     print("*** Reducing DL0 to DL1 data***")