diff --git a/src/hidrocl/__init__.py b/src/hidrocl/__init__.py index 060af67..3a84075 100644 --- a/src/hidrocl/__init__.py +++ b/src/hidrocl/__init__.py @@ -32,6 +32,7 @@ except ImportError: print("ImportError") + def reload_paths(): """ Reloads the paths module @@ -106,6 +107,7 @@ def set_forecasted_products_path(path): reload_paths() return None + def prepare_path(path): """ Check if the path exists and if not, creates it diff --git a/src/hidrocl/__version__.py b/src/hidrocl/__version__.py index 516c6f0..a7238bb 100644 --- a/src/hidrocl/__version__.py +++ b/src/hidrocl/__version__.py @@ -1,3 +1,3 @@ # coding=utf-8 -__version__ = '0.0.24' +__version__ = '0.0.25' diff --git a/src/hidrocl/download/__init__.py b/src/hidrocl/download/__init__.py index 3b24dd3..d996883 100644 --- a/src/hidrocl/download/__init__.py +++ b/src/hidrocl/download/__init__.py @@ -91,20 +91,17 @@ def download_era5land(year, month, day, path, timeout=60, retry_max=10, sleep_ma os.remove(fname) - ds0 = xr.open_dataset(os.path.join(pth,'data_0.nc')) - ds1 = xr.open_dataset(os.path.join(pth,'data_1.nc')) - ds2 = xr.open_dataset(os.path.join(pth,'data_2.nc')) + ds0 = xr.open_dataset(os.path.join(pth, 'data_0.nc')) + ds1 = xr.open_dataset(os.path.join(pth, 'data_1.nc')) + ds2 = xr.open_dataset(os.path.join(pth, 'data_2.nc')) - ds = xr.merge([ds0, ds1, ds2], join = 'override') + ds = xr.merge([ds0, ds1, ds2], join='override') ds = ds.drop_vars(['number', 'expver']) ds.to_netcdf(fnameout) shutil.rmtree(pth) - - - def download_era5(year, month, day, path): """function to download era5 reanalysis data from CDS @@ -128,7 +125,6 @@ def download_era5(year, month, day, path): fname = os.path.join(path, f'era5_{year:04d}{month:02d}{day:02d}.nc') - dataset = 'reanalysis-era5-single-levels' request = { @@ -535,7 +531,8 @@ def earthdata_download(what, product_path, start, end): print('Downloaded finished') -def download_pdirnow(start, end, product_path, check_ppath = False): + +def download_pdirnow(start, end, product_path, check_ppath=False): """ Download PDIRNow data from CHRS FTP server. @@ -555,7 +552,9 @@ def download_pdirnow(start, end, product_path, check_ppath = False): ftp_server = 'persiann.eng.uci.edu' ftp_path = 'CHRSdata/PDIRNow/PDIRNowdaily' - while True: + i = 0 + + while i < 5: try: ftp = ftplib.FTP(ftp_server) ftp.login() @@ -564,8 +563,13 @@ def download_pdirnow(start, end, product_path, check_ppath = False): except: print('FTP connection failed. Trying again in 5 seconds...') time.sleep(5) + i += 1 continue + if i == 5: + print('FTP connection failed. Please try again later') + return + dir_list = [] ftp.dir(dir_list.append) files_list = [value.split(' ')[-1] for value in dir_list if 'bin' in value] diff --git a/src/hidrocl/download/tools.py b/src/hidrocl/download/tools.py index 0083665..8b7130f 100644 --- a/src/hidrocl/download/tools.py +++ b/src/hidrocl/download/tools.py @@ -18,4 +18,4 @@ def __exit__(self, exc_type, exc_val, exc_tb): sys.stdout.close() sys.stderr.close() sys.stdout = self._original_stdout - sys.stderr = self._original_stderr \ No newline at end of file + sys.stderr = self._original_stderr diff --git a/src/hidrocl/products/__init__.py b/src/hidrocl/products/__init__.py index a8d911d..9a1204c 100644 --- a/src/hidrocl/products/__init__.py +++ b/src/hidrocl/products/__init__.py @@ -1114,13 +1114,13 @@ def run_extraction(self, limit=None): aggregation=self.agg, value=4) if scene not in self.dnf.indatabase: e.zonal_stats(scene, scenes_path, - temp_dir, 'lulc_dnf', - self.dnf.catchment_names, self.var_log, - database=self.dnf.database, - pcdatabase=self.dnf.pcdatabase, - vector_path=self.vectorpath, - layer="LC_Type1", - aggregation=self.agg, value=3) + temp_dir, 'lulc_dnf', + self.dnf.catchment_names, self.var_log, + database=self.dnf.database, + pcdatabase=self.dnf.pcdatabase, + vector_path=self.vectorpath, + layer="LC_Type1", + aggregation=self.agg, value=3) if scene not in self.ebf.indatabase: e.zonal_stats(scene, scenes_path, temp_dir, 'lulc_ebf', @@ -1221,7 +1221,6 @@ def run_extraction(self, limit=None): layer="LC_Type1", aggregation=self.agg, value=8) - def run_maintainer(self, log_file, limit=None): """ Run file maintainer. It will remove any file with problems diff --git a/src/hidrocl/products/extractions.py b/src/hidrocl/products/extractions.py index 3697caa..1f29ee1 100644 --- a/src/hidrocl/products/extractions.py +++ b/src/hidrocl/products/extractions.py @@ -20,7 +20,6 @@ from rioxarray import exceptions as rxre from rioxarray.merge import merge_arrays - path = Path(__file__).parent.absolute() debug = False @@ -187,8 +186,8 @@ def load_persiann(file): )) persiann.coords['lon'] = (persiann.coords['lon'] + 180) % 360 - 180 - return persiann.sortby(persiann.lon)\ - .sortby(persiann.lat).sel(lat=slice(-55, -15), lon=slice(-75, -65))\ + return persiann.sortby(persiann.lon) \ + .sortby(persiann.lat).sel(lat=slice(-55, -15), lon=slice(-75, -65)) \ .rename({'lon': 'x', 'lat': 'y'}) @@ -209,8 +208,8 @@ def load_gfs(file, var, day=0): da = xarray.open_dataset(file, mask_and_scale=True) da = da[var] da.load() - da = da.sel(valid_time=slice(da.time+pd.to_timedelta(24*day + 3, unit='H'), - da.time+pd.to_timedelta(24*day + 26, unit='H')))\ + da = da.sel(valid_time=slice(da.time + pd.to_timedelta(24 * day + 3, unit='H'), + da.time + pd.to_timedelta(24 * day + 26, unit='H'))) \ .transpose('valid_time', 'latitude', 'longitude') da.coords['longitude'] = (da.coords['longitude'] + 180) % 360 - 180 return da @@ -434,8 +433,8 @@ def write_line2(database, result, catchment_names, file_id, file_date, ncol=1): Returns: None """ - gauge_id_result = result.iloc[:,0].astype(str).tolist() - value_result = result.iloc[:,ncol].astype(str).tolist() + gauge_id_result = result.iloc[:, 0].astype(str).tolist() + value_result = result.iloc[:, ncol].astype(str).tolist() if catchment_names == gauge_id_result: value_result.insert(0, file_id) @@ -483,10 +482,10 @@ def extract_data(v, r, fun, debug=False, debug_path=None, name=None): gdf = gpd.read_file(v) result_val = exactextract.exact_extract(r, gdf, fun, include_cols='gauge_id', output='pandas', - max_cells_in_memory = 300000000) + max_cells_in_memory=300000000) result_pc = exactextract.exact_extract(xarray.where(r.isnull(), 0, 1), gdf, 'mean', include_cols='gauge_id', output='pandas', - max_cells_in_memory = 300000000) + max_cells_in_memory=300000000) result_pc.iloc[:, 1:] = result_pc.iloc[:, 1:] * 1000 result_val = result_val.merge(result_pc, on='gauge_id', how='left') result_val = result_val.fillna(0).round(0).astype(int) @@ -604,7 +603,7 @@ def zonal_stats(scene, scenes_path, tempfolder, name, mos = xarray.where(mos == val, 1, 0) agg = kwargs.get("aggregation") if agg == "mean": - mos = mos*1000 + mos = mos * 1000 except (rxre.RioXarrayError, rioe.RasterioIOError): return print(f"Error in scene {scene}") @@ -652,7 +651,7 @@ def zonal_stats(scene, scenes_path, tempfolder, name, elif kwargs.get("layer") == 'gh': pass elif kwargs.get("layer") == 't2m': - mos_pre = (mos_pre - 273.15)*10 + mos_pre = (mos_pre - 273.15) * 10 else: mos_pre = mos_pre * 10 mos_list.append(mos_pre) @@ -808,13 +807,13 @@ def zonal_stats(scene, scenes_path, tempfolder, name, match name: case 'snow': result_df = extract_data(kwargs.get("north_vector_path"), mos, 'mean', debug=debug, - debug_path=debug_path, name='n_'+result_file) + debug_path=debug_path, name='n_' + result_file) write_line2(kwargs.get("north_database"), result_df, catchment_names, scene, file_date, ncol=1) write_line2(kwargs.get("north_pcdatabase"), result_df, catchment_names, scene, file_date, ncol=2) result_df = extract_data(kwargs.get("south_vector_path"), mos, 'mean', debug=debug, - debug_path=debug_path, name='s_'+result_file) + debug_path=debug_path, name='s_' + result_file) write_line2(kwargs.get("south_database"), result_df, catchment_names, scene, file_date, ncol=1) write_line2(kwargs.get("south_pcdatabase"), result_df, catchment_names, scene, file_date, ncol=2) @@ -860,7 +859,7 @@ def zonal_stats(scene, scenes_path, tempfolder, name, case _: result_df = extract_data(kwargs.get("vector_path"), mos, 'mean', debug=debug, - debug_path=debug_path, name= result_file) + debug_path=debug_path, name=result_file) write_line2(kwargs.get("database"), result_df, catchment_names, scene, file_date, ncol=1) write_line2(kwargs.get("pcdatabase"), result_df, catchment_names, scene, file_date, ncol=2) diff --git a/src/hidrocl/variables/methods.py b/src/hidrocl/variables/methods.py index 39ffd25..89e5cc5 100644 --- a/src/hidrocl/variables/methods.py +++ b/src/hidrocl/variables/methods.py @@ -5,7 +5,7 @@ import matplotlib.pyplot as plt -def checkdatabase(database, catchment_names=None, supportcreation = None): +def checkdatabase(database, catchment_names=None, supportcreation=None): """ Check if the database exists and is valid @@ -31,7 +31,6 @@ def checkdatabase(database, catchment_names=None, supportcreation = None): else: raise ImportError('Database not found. Check filename or create it') - else: print('Database not found, creating it for ' + database) header_line = [str(s) for s in catchment_names]