Updated download functions and some adjustments for PEP formatting

src/hidrocl/__init__.py

@@ -32,6 +32,7 @@
 except ImportError:
     print("ImportError")
 
+
 def reload_paths():
     """
     Reloads the paths module
@@ -106,6 +107,7 @@ def set_forecasted_products_path(path):
     reload_paths()
     return None
 
+
 def prepare_path(path):
     """
     Check if the path exists and if not, creates it

src/hidrocl/__version__.py

@@ -1,3 +1,3 @@
 # coding=utf-8
 
-__version__ = '0.0.24'
+__version__ = '0.0.25'

src/hidrocl/download/__init__.py

@@ -91,20 +91,17 @@ def download_era5land(year, month, day, path, timeout=60, retry_max=10, sleep_ma
 
     os.remove(fname)
 
-    ds0 = xr.open_dataset(os.path.join(pth,'data_0.nc'))
-    ds1 = xr.open_dataset(os.path.join(pth,'data_1.nc'))
-    ds2 = xr.open_dataset(os.path.join(pth,'data_2.nc'))
+    ds0 = xr.open_dataset(os.path.join(pth, 'data_0.nc'))
+    ds1 = xr.open_dataset(os.path.join(pth, 'data_1.nc'))
+    ds2 = xr.open_dataset(os.path.join(pth, 'data_2.nc'))
 
-    ds = xr.merge([ds0, ds1, ds2], join = 'override')
+    ds = xr.merge([ds0, ds1, ds2], join='override')
     ds = ds.drop_vars(['number', 'expver'])
     ds.to_netcdf(fnameout)
 
     shutil.rmtree(pth)
 
 
-
-
-
 def download_era5(year, month, day, path):
     """function to download era5 reanalysis data from CDS
 
@@ -128,7 +125,6 @@ def download_era5(year, month, day, path):
 
     fname = os.path.join(path, f'era5_{year:04d}{month:02d}{day:02d}.nc')
 
-
     dataset = 'reanalysis-era5-single-levels'
 
     request = {
@@ -535,7 +531,8 @@ def earthdata_download(what, product_path, start, end):
 
     print('Downloaded finished')
 
-def download_pdirnow(start, end, product_path, check_ppath = False):
+
+def download_pdirnow(start, end, product_path, check_ppath=False):
     """
     Download PDIRNow data from CHRS FTP server.
 
@@ -555,7 +552,9 @@ def download_pdirnow(start, end, product_path, check_ppath = False):
     ftp_server = 'persiann.eng.uci.edu'
     ftp_path = 'CHRSdata/PDIRNow/PDIRNowdaily'
 
-    while True:
+    i = 0
+
+    while i < 5:
         try:
             ftp = ftplib.FTP(ftp_server)
             ftp.login()
@@ -564,8 +563,13 @@ def download_pdirnow(start, end, product_path, check_ppath = False):
         except:
             print('FTP connection failed. Trying again in 5 seconds...')
             time.sleep(5)
+            i += 1
             continue
 
+    if i == 5:
+        print('FTP connection failed. Please try again later')
+        return
+
     dir_list = []
     ftp.dir(dir_list.append)
     files_list = [value.split(' ')[-1] for value in dir_list if 'bin' in value]

src/hidrocl/download/tools.py

@@ -18,4 +18,4 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         sys.stdout.close()
         sys.stderr.close()
         sys.stdout = self._original_stdout
-        sys.stderr = self._original_stderr
+        sys.stderr = self._original_stderr

src/hidrocl/products/__init__.py

@@ -1114,13 +1114,13 @@ def run_extraction(self, limit=None):
                                   aggregation=self.agg, value=4)
                 if scene not in self.dnf.indatabase:
                     e.zonal_stats(scene, scenes_path,
-                                    temp_dir, 'lulc_dnf',
-                                    self.dnf.catchment_names, self.var_log,
-                                    database=self.dnf.database,
-                                    pcdatabase=self.dnf.pcdatabase,
-                                    vector_path=self.vectorpath,
-                                    layer="LC_Type1",
-                                    aggregation=self.agg, value=3)
+                                  temp_dir, 'lulc_dnf',
+                                  self.dnf.catchment_names, self.var_log,
+                                  database=self.dnf.database,
+                                  pcdatabase=self.dnf.pcdatabase,
+                                  vector_path=self.vectorpath,
+                                  layer="LC_Type1",
+                                  aggregation=self.agg, value=3)
                 if scene not in self.ebf.indatabase:
                     e.zonal_stats(scene, scenes_path,
                                   temp_dir, 'lulc_ebf',
@@ -1221,7 +1221,6 @@ def run_extraction(self, limit=None):
                                   layer="LC_Type1",
                                   aggregation=self.agg, value=8)
 
-
     def run_maintainer(self, log_file, limit=None):
         """
         Run file maintainer. It will remove any file with problems

src/hidrocl/products/extractions.py

@@ -20,7 +20,6 @@
 from rioxarray import exceptions as rxre
 from rioxarray.merge import merge_arrays
 
-
 path = Path(__file__).parent.absolute()
 
 debug = False
@@ -187,8 +186,8 @@ def load_persiann(file):
                                     ))
 
         persiann.coords['lon'] = (persiann.coords['lon'] + 180) % 360 - 180
-        return persiann.sortby(persiann.lon)\
-            .sortby(persiann.lat).sel(lat=slice(-55, -15), lon=slice(-75, -65))\
+        return persiann.sortby(persiann.lon) \
+            .sortby(persiann.lat).sel(lat=slice(-55, -15), lon=slice(-75, -65)) \
             .rename({'lon': 'x', 'lat': 'y'})
 
 
@@ -209,8 +208,8 @@ def load_gfs(file, var, day=0):
         da = xarray.open_dataset(file, mask_and_scale=True)
         da = da[var]
         da.load()
-        da = da.sel(valid_time=slice(da.time+pd.to_timedelta(24*day + 3, unit='H'),
-                                     da.time+pd.to_timedelta(24*day + 26, unit='H')))\
+        da = da.sel(valid_time=slice(da.time + pd.to_timedelta(24 * day + 3, unit='H'),
+                                     da.time + pd.to_timedelta(24 * day + 26, unit='H'))) \
             .transpose('valid_time', 'latitude', 'longitude')
         da.coords['longitude'] = (da.coords['longitude'] + 180) % 360 - 180
         return da
@@ -434,8 +433,8 @@ def write_line2(database, result, catchment_names, file_id, file_date, ncol=1):
     Returns:
         None
     """
-    gauge_id_result = result.iloc[:,0].astype(str).tolist()
-    value_result = result.iloc[:,ncol].astype(str).tolist()
+    gauge_id_result = result.iloc[:, 0].astype(str).tolist()
+    value_result = result.iloc[:, ncol].astype(str).tolist()
 
     if catchment_names == gauge_id_result:
         value_result.insert(0, file_id)
@@ -483,10 +482,10 @@ def extract_data(v, r, fun, debug=False, debug_path=None, name=None):
     gdf = gpd.read_file(v)
 
     result_val = exactextract.exact_extract(r, gdf, fun, include_cols='gauge_id', output='pandas',
-                                            max_cells_in_memory = 300000000)
+                                            max_cells_in_memory=300000000)
     result_pc = exactextract.exact_extract(xarray.where(r.isnull(), 0, 1), gdf, 'mean',
                                            include_cols='gauge_id', output='pandas',
-                                           max_cells_in_memory = 300000000)
+                                           max_cells_in_memory=300000000)
     result_pc.iloc[:, 1:] = result_pc.iloc[:, 1:] * 1000
     result_val = result_val.merge(result_pc, on='gauge_id', how='left')
     result_val = result_val.fillna(0).round(0).astype(int)
@@ -604,7 +603,7 @@ def zonal_stats(scene, scenes_path, tempfolder, name,
                 mos = xarray.where(mos == val, 1, 0)
                 agg = kwargs.get("aggregation")
                 if agg == "mean":
-                    mos = mos*1000
+                    mos = mos * 1000
 
             except (rxre.RioXarrayError, rioe.RasterioIOError):
                 return print(f"Error in scene {scene}")
@@ -652,7 +651,7 @@ def zonal_stats(scene, scenes_path, tempfolder, name,
                         elif kwargs.get("layer") == 'gh':
                             pass
                         elif kwargs.get("layer") == 't2m':
-                            mos_pre = (mos_pre - 273.15)*10
+                            mos_pre = (mos_pre - 273.15) * 10
                         else:
                             mos_pre = mos_pre * 10
                         mos_list.append(mos_pre)
@@ -808,13 +807,13 @@ def zonal_stats(scene, scenes_path, tempfolder, name,
     match name:
         case 'snow':
             result_df = extract_data(kwargs.get("north_vector_path"), mos, 'mean', debug=debug,
-                                       debug_path=debug_path, name='n_'+result_file)
+                                     debug_path=debug_path, name='n_' + result_file)
 
             write_line2(kwargs.get("north_database"), result_df, catchment_names, scene, file_date, ncol=1)
             write_line2(kwargs.get("north_pcdatabase"), result_df, catchment_names, scene, file_date, ncol=2)
 
             result_df = extract_data(kwargs.get("south_vector_path"), mos, 'mean', debug=debug,
-                                       debug_path=debug_path, name='s_'+result_file)
+                                     debug_path=debug_path, name='s_' + result_file)
 
             write_line2(kwargs.get("south_database"), result_df, catchment_names, scene, file_date, ncol=1)
             write_line2(kwargs.get("south_pcdatabase"), result_df, catchment_names, scene, file_date, ncol=2)
@@ -860,7 +859,7 @@ def zonal_stats(scene, scenes_path, tempfolder, name,
 
         case _:
             result_df = extract_data(kwargs.get("vector_path"), mos, 'mean', debug=debug,
-                                     debug_path=debug_path, name= result_file)
+                                     debug_path=debug_path, name=result_file)
 
             write_line2(kwargs.get("database"), result_df, catchment_names, scene, file_date, ncol=1)
             write_line2(kwargs.get("pcdatabase"), result_df, catchment_names, scene, file_date, ncol=2)

src/hidrocl/variables/methods.py

@@ -5,7 +5,7 @@
 import matplotlib.pyplot as plt
 
 
-def checkdatabase(database, catchment_names=None, supportcreation = None):
+def checkdatabase(database, catchment_names=None, supportcreation=None):
     """
     Check if the database exists and is valid
 
@@ -31,7 +31,6 @@ def checkdatabase(database, catchment_names=None, supportcreation = None):
             else:
                 raise ImportError('Database not found. Check filename or create it')
 
-
         else:
             print('Database not found, creating it for ' + database)
             header_line = [str(s) for s in catchment_names]