From 5d980b4c569b75998fb13aebe3ceb9f43408b4c8 Mon Sep 17 00:00:00 2001 From: hhs732 Date: Fri, 10 May 2024 21:16:47 +0000 Subject: [PATCH 1/7] analyze_for_missing_FIM_cells.py added --- config/params_template.env | 5 + src/delineate_hydros_and_produce_HAND.sh | 12 ++ tools/analyze_for_missing_FIM_cells.py | 150 +++++++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 tools/analyze_for_missing_FIM_cells.py diff --git a/config/params_template.env b/config/params_template.env index ab761336c..1e8a71b53 100644 --- a/config/params_template.env +++ b/config/params_template.env @@ -65,3 +65,8 @@ export manual_calb_toggle="True" export ncores_gw=1 # mpi number of cores for gagewatershed export ncores_fd=1 # mpi number of cores for flow directions export memfree=0G # min free memory required to start a new job or keep youngest job alive + +#### Healed HAND #### +# Removes Hydro-conditioning Artifacts +# by subtracting dem_thalweg_cond.tif from HAND_rem + DEM (true=on; false=off) +export healed_hand_hydrocondition=true \ No newline at end of file diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh index 379761f1a..68514f221 100755 --- a/src/delineate_hydros_and_produce_HAND.sh +++ b/src/delineate_hydros_and_produce_HAND.sh @@ -229,6 +229,18 @@ $taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$curr -table $tempCurrentBranchDataDir/src_base_$current_branch_id.csv +## HEAL HAND -- REMOVES HYDROCONDITIONING ARTIFACTS ## +if [ "$healed_hand_hydrocondition" = true ]; then + echo + echo -e $startDiv"Healed HAND to Remove Hydro-conditioning Artifacts $hucNumber $current_branch_id" + gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \ + -R $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \ + -D $tempCurrentBranchDataDir/dem_meters_$current_branch_id.tif \ + -T $tempCurrentBranchDataDir/dem_thalwegCond_$current_branch_id.tif \ + --calc="R+(D-T)" --NoDataValue=$ndv \ + --outfile=$tempCurrentBranchDataDir/"rem_zeroed_masked_$current_branch_id.tif" +fi + ## HEAL HAND BRIDGES (note resolution is set to 10m) # May or may not be a bridge file, depends if the HUC has an applicble one. diff --git a/tools/analyze_for_missing_FIM_cells.py b/tools/analyze_for_missing_FIM_cells.py new file mode 100644 index 000000000..33b200657 --- /dev/null +++ b/tools/analyze_for_missing_FIM_cells.py @@ -0,0 +1,150 @@ +""" +what is the purpose, example usage, link to the PR + +""" + +import argparse +import os +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import rasterio +from rasterio import plot as rioplot + + +# This will make jupyter display all columns of the hydroTable +pd.options.display.max_columns = None + +# ------------------------------------------- +# fim_dir_str = "/home/rdp-user/outputs/healded_fim_hca_01080107_if2/" +# fim_dir_str = Path(fim_dir_str) +# huc8_num = "01080107" + + +def find_missing_fim_cells(fim_dir_str, huc8_num, dir_fig): + + fim_dir = Path(fim_dir_str) + branch0_dir = Path(fim_dir, huc8_num, "branches", "0") + hydroTable_csv = Path(branch0_dir, "hydroTable_0.csv") + hydroTable = pd.read_csv(hydroTable_csv) + + branch0_streams = hydroTable[(hydroTable["order_"] == 1) | (hydroTable["order_"] == 2)] + branch0_hydroids = list(branch0_streams['HydroID'].drop_duplicates(keep='first')) + + rem_tif = Path(branch0_dir, "rem_zeroed_masked_0.tif") + catchments_tif = Path(branch0_dir, "gw_catchments_reaches_filtered_addedAttributes_0.tif") + + with rasterio.open(catchments_tif) as catchments: + catchments = catchments.read(1) + + with rasterio.open(rem_tif) as rem: + # rem_nodata = rem.profile['nodata'] + rem_extent = rioplot.plotting_extent(rem) + # rem_transform = rem.transform + # rem_crs = rem.crs + rem = rem.read(1) + + # Filter the REM to the catchments of interest + rem = np.where(~np.isin(catchments, branch0_hydroids), np.nan, rem) + + # Filter the REM again to only the 0 values + rem_zeros = rem.copy() + rem_zeros[np.where(rem_zeros != 0)] = np.nan + + cells_remaining = np.count_nonzero(~np.isnan(rem_zeros)) + print(f"Actual count of cells remaining: {cells_remaining}") + + non_zero_count = (catchments != 0).sum() + percentage_b0_rem0 = cells_remaining / non_zero_count + print(f"Actual percentage of branch0 stream cells that have not inundated: {percentage_b0_rem0}") + + # Finding branch0 hydroIDs that do not have zero rem (never inundate) + target_hydroids = [] + for hydroid in branch0_hydroids: + + rem_hydroid = rem.copy() + hydroid_ls = [hydroid] + rem_hydroid = np.where(~np.isin(catchments, hydroid_ls), np.nan, rem_hydroid) + + rem_hydroid_nonnan = rem_hydroid[~np.isnan(rem_hydroid)] + cond = min(rem_hydroid_nonnan) + + if cond > 0: + target_hydroids.append(hydroid) + + print( + f"{len(target_hydroids)} catchments in HUC {huc8_num} do not inundate, including HUC8s {target_hydroids}" + ) + + # # finding branch0 hydroIDs that have zero rem + # catchments_rem0 = catchments.copy() + # catchments_rem0 = np.where(np.isnan(rem_zeros), np.nan, catchments_rem0) + + # catchments_rem0_ls = catchments_rem0[~np.isnan(catchments_rem0)].tolist() + # hydroids_rem0_ls = list(set(catchments_rem0_ls)) + # hydroids_rem0 = [int(x) for x in hydroids_rem0_ls] + + # # finding inundated branch0 rem and hydroids + # rem_inund = rem.copy() + # rem_inund[np.where(rem_inund <= 0)] = np.nan + + # catchments_inund = catchments.copy() + # catchments_inund = np.where(np.isnan(rem_inund), np.nan, catchments_inund) + + # catchments_inund_ls = catchments_inund[~np.isnan(catchments_inund)].tolist() + # hydroids_inund_ls = list(set(catchments_inund_ls)) + # hydroids_inund = [int(x) for x in hydroids_inund_ls] + + # hydroIds_branch0_noinund = len(hydroids_inund) - len(list(set(hydroids_inund) & set(hydroids_rem0))) + + # Simple Plotting + # reaches_gpkg = Path(branch0_dir, "demDerived_reaches_split_filtered_addedAttributes_crosswalked_0.gpkg") + # reaches = gpd.read_file(reaches_gpkg) + # plot_bounds = reaches.loc[reaches.HydroID.astype(int) == hydroid].bounds.iloc[0] + + # Plot the zero REM cells + fig = plt.figure(figsize=(12, 7)) + ax = fig.subplots(1, 1) + + # reaches.plot(ax=ax, color='brown') + im = ax.imshow(rem_zeros, cmap='bwr', extent=rem_extent, interpolation='none') + + path_fig = Path(dir_fig, 'rem_zeros.png') + plt.savefig(im, path_fig) + + +if __name__ == "__main__": + + # Parse arguments. + parser = argparse.ArgumentParser(description="Analysis for missing FIM cells.") + parser.add_argument( + "-r", + dest="fim_dir_str", + help="Path to directory storing FIM outputs. Type = string", + required=True, + type=str, + ) + parser.add_argument( + "-w", dest="huc8_num", help="HUC8 that is being analyzsd. Type = string", required=True, type=str + ) + parser.add_argument( + "-o", dest="dir_fig", help="Path to save rem_zero. Type = string", required=True, type=str + ) + + # Assign variables from arguments. + args = vars(parser.parse_args()) + + fim_dir = args["fim_dir_str"] + huc8_num = args["huc8_num"] + dir_fig = args["dir_fig"] + + if not os.path.exists(fim_dir): + print("FIM directory: " + fim_dir + " does not exist.") + quit + + if not os.path.exists(dir_fig): + os.mkdir(dir_fig) + + find_missing_fim_cells(fim_dir, huc8_num, dir_fig) From 92e67152e6908d8e483dcfeeeec78d01e47c678b Mon Sep 17 00:00:00 2001 From: hhs732 Date: Mon, 13 May 2024 18:15:05 +0000 Subject: [PATCH 2/7] newest hca removing --- tools/analyze_for_missing_FIM_cells.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/analyze_for_missing_FIM_cells.py b/tools/analyze_for_missing_FIM_cells.py index 33b200657..2f2a2e50d 100644 --- a/tools/analyze_for_missing_FIM_cells.py +++ b/tools/analyze_for_missing_FIM_cells.py @@ -41,7 +41,7 @@ def find_missing_fim_cells(fim_dir_str, huc8_num, dir_fig): with rasterio.open(rem_tif) as rem: # rem_nodata = rem.profile['nodata'] - rem_extent = rioplot.plotting_extent(rem) + # rem_extent = rioplot.plotting_extent(rem) # rem_transform = rem.transform # rem_crs = rem.crs rem = rem.read(1) @@ -78,6 +78,7 @@ def find_missing_fim_cells(fim_dir_str, huc8_num, dir_fig): f"{len(target_hydroids)} catchments in HUC {huc8_num} do not inundate, including HUC8s {target_hydroids}" ) + """ # # finding branch0 hydroIDs that have zero rem # catchments_rem0 = catchments.copy() # catchments_rem0 = np.where(np.isnan(rem_zeros), np.nan, catchments_rem0) @@ -114,6 +115,10 @@ def find_missing_fim_cells(fim_dir_str, huc8_num, dir_fig): path_fig = Path(dir_fig, 'rem_zeros.png') plt.savefig(im, path_fig) + """ + + return [target_hydroids, cells_remaining, percentage_b0_rem0] + if __name__ == "__main__": From e5e1fa8cbe9d956df9176d4b09d35f83a5bff6a6 Mon Sep 17 00:00:00 2001 From: hhs732 Date: Mon, 13 May 2024 18:23:38 +0000 Subject: [PATCH 3/7] error fixed --- config/params_template.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/params_template.env b/config/params_template.env index 1e8a71b53..2c475ab9b 100644 --- a/config/params_template.env +++ b/config/params_template.env @@ -69,4 +69,4 @@ export memfree=0G # min free memory required to start a new job or keep youngest #### Healed HAND #### # Removes Hydro-conditioning Artifacts # by subtracting dem_thalweg_cond.tif from HAND_rem + DEM (true=on; false=off) -export healed_hand_hydrocondition=true \ No newline at end of file +export healed_hand_hydrocondition=true From 8b2bd0ef4bf22d702efa277d75c23543b2a9389d Mon Sep 17 00:00:00 2001 From: hhs732 Date: Tue, 14 May 2024 17:49:50 +0000 Subject: [PATCH 4/7] hydro-conditioning tool updated --- config/params_template.env | 9 +- src/delineate_hydros_and_produce_HAND.sh | 1 - tools/analyze_for_missing_FIM_cells.py | 177 ++++++++++++++--------- 3 files changed, 111 insertions(+), 76 deletions(-) diff --git a/config/params_template.env b/config/params_template.env index 2c475ab9b..8312bea92 100644 --- a/config/params_template.env +++ b/config/params_template.env @@ -31,6 +31,10 @@ export branch_zero_id="0" export mask_leveed_area_toggle=True # Toggle to mask levee-protected areas from DEM export levee_id_attribute=SYSTEM_ID +#### Healed HAND #### +# Removes Hydro-conditioning Artifacts (true=on; false=off) +export healed_hand_hydrocondition=true + #### apply bathymetry adjustment to rating curve #### export bathymetry_adjust=True @@ -65,8 +69,3 @@ export manual_calb_toggle="True" export ncores_gw=1 # mpi number of cores for gagewatershed export ncores_fd=1 # mpi number of cores for flow directions export memfree=0G # min free memory required to start a new job or keep youngest job alive - -#### Healed HAND #### -# Removes Hydro-conditioning Artifacts -# by subtracting dem_thalweg_cond.tif from HAND_rem + DEM (true=on; false=off) -export healed_hand_hydrocondition=true diff --git a/src/delineate_hydros_and_produce_HAND.sh b/src/delineate_hydros_and_produce_HAND.sh index 68514f221..efe6e5f55 100755 --- a/src/delineate_hydros_and_produce_HAND.sh +++ b/src/delineate_hydros_and_produce_HAND.sh @@ -231,7 +231,6 @@ $taudemDir/catchhydrogeo -hand $tempCurrentBranchDataDir/rem_zeroed_masked_$curr ## HEAL HAND -- REMOVES HYDROCONDITIONING ARTIFACTS ## if [ "$healed_hand_hydrocondition" = true ]; then - echo echo -e $startDiv"Healed HAND to Remove Hydro-conditioning Artifacts $hucNumber $current_branch_id" gdal_calc.py --quiet --type=Float32 --overwrite --co "COMPRESS=LZW" --co "BIGTIFF=YES" --co "TILED=YES" \ -R $tempCurrentBranchDataDir/rem_zeroed_masked_$current_branch_id.tif \ diff --git a/tools/analyze_for_missing_FIM_cells.py b/tools/analyze_for_missing_FIM_cells.py index 2f2a2e50d..766cae4b4 100644 --- a/tools/analyze_for_missing_FIM_cells.py +++ b/tools/analyze_for_missing_FIM_cells.py @@ -11,114 +11,151 @@ import numpy as np import pandas as pd import rasterio -from rasterio import plot as rioplot + + +# from rasterio import plot as rioplot # This will make jupyter display all columns of the hydroTable pd.options.display.max_columns = None -# ------------------------------------------- -# fim_dir_str = "/home/rdp-user/outputs/healded_fim_hca_01080107_if2/" -# fim_dir_str = Path(fim_dir_str) -# huc8_num = "01080107" - -def find_missing_fim_cells(fim_dir_str, huc8_num, dir_fig): +# ------------------------------------------------------ +def find_missing_fim_cells_for1huc(fim_dir, huc8_num): - fim_dir = Path(fim_dir_str) + fim_dir = Path(fim_dir) branch0_dir = Path(fim_dir, huc8_num, "branches", "0") hydroTable_csv = Path(branch0_dir, "hydroTable_0.csv") hydroTable = pd.read_csv(hydroTable_csv) - branch0_streams = hydroTable[(hydroTable["order_"] == 1) | (hydroTable["order_"] == 2)] - branch0_hydroids = list(branch0_streams['HydroID'].drop_duplicates(keep='first')) - rem_tif = Path(branch0_dir, "rem_zeroed_masked_0.tif") catchments_tif = Path(branch0_dir, "gw_catchments_reaches_filtered_addedAttributes_0.tif") with rasterio.open(catchments_tif) as catchments: catchments = catchments.read(1) - with rasterio.open(rem_tif) as rem: - # rem_nodata = rem.profile['nodata'] - # rem_extent = rioplot.plotting_extent(rem) - # rem_transform = rem.transform - # rem_crs = rem.crs - rem = rem.read(1) + stream_orders = list(hydroTable["order_"].drop_duplicates(keep='first')) + stream_orders = sorted(stream_orders) - # Filter the REM to the catchments of interest - rem = np.where(~np.isin(catchments, branch0_hydroids), np.nan, rem) + analysis_data_ordr = [] + for ordr in stream_orders: - # Filter the REM again to only the 0 values - rem_zeros = rem.copy() - rem_zeros[np.where(rem_zeros != 0)] = np.nan + branch0_streams = hydroTable[(hydroTable["order_"] == ordr)] + num_streams_order = len(branch0_streams) + branch0_hydroids = list(branch0_streams['HydroID'].drop_duplicates(keep='first')) - cells_remaining = np.count_nonzero(~np.isnan(rem_zeros)) - print(f"Actual count of cells remaining: {cells_remaining}") + with rasterio.open(rem_tif) as rem: + rem = rem.read(1) - non_zero_count = (catchments != 0).sum() - percentage_b0_rem0 = cells_remaining / non_zero_count - print(f"Actual percentage of branch0 stream cells that have not inundated: {percentage_b0_rem0}") + # Filter the REM to the catchments of interest + rem = np.where(~np.isin(catchments, branch0_hydroids), np.nan, rem) - # Finding branch0 hydroIDs that do not have zero rem (never inundate) - target_hydroids = [] - for hydroid in branch0_hydroids: + # Filter the REM again to only the 0 values + rem_zeros = rem.copy() + rem_zeros[np.where(rem_zeros != 0)] = np.nan - rem_hydroid = rem.copy() - hydroid_ls = [hydroid] - rem_hydroid = np.where(~np.isin(catchments, hydroid_ls), np.nan, rem_hydroid) + cells_remaining = np.count_nonzero(~np.isnan(rem_zeros)) + print("") + print(f"Analyzing Branch0, {ordr}-order streams FIM in HUC8 {huc8_num}") + print("") + print(f"Actual count of cells remaining: {cells_remaining}") - rem_hydroid_nonnan = rem_hydroid[~np.isnan(rem_hydroid)] - cond = min(rem_hydroid_nonnan) + non_zero_count = (catchments != 0).sum() + percentage_b0_rem0 = round(100 * (cells_remaining / non_zero_count), 4) + print("") + print( + f"Actual percentage of branch0, {ordr}-order stream cells that have not inundated: {percentage_b0_rem0}%" + ) - if cond > 0: - target_hydroids.append(hydroid) + # Finding branch0 hydroIDs that do not have zero rem (never inundate, notches) + target_hydroids = [] + for hydroid in branch0_hydroids: - print( - f"{len(target_hydroids)} catchments in HUC {huc8_num} do not inundate, including HUC8s {target_hydroids}" - ) + rem_hydroid = rem.copy() + hydroid_ls = [hydroid] + rem_hydroid = np.where(~np.isin(catchments, hydroid_ls), np.nan, rem_hydroid) - """ - # # finding branch0 hydroIDs that have zero rem - # catchments_rem0 = catchments.copy() - # catchments_rem0 = np.where(np.isnan(rem_zeros), np.nan, catchments_rem0) + rem_hydroid_nonnan = rem_hydroid[~np.isnan(rem_hydroid)] + cond = min(rem_hydroid_nonnan) - # catchments_rem0_ls = catchments_rem0[~np.isnan(catchments_rem0)].tolist() - # hydroids_rem0_ls = list(set(catchments_rem0_ls)) - # hydroids_rem0 = [int(x) for x in hydroids_rem0_ls] + if cond > 0: + target_hydroids.append(hydroid) - # # finding inundated branch0 rem and hydroids - # rem_inund = rem.copy() - # rem_inund[np.where(rem_inund <= 0)] = np.nan + print("") + print( + f"{len(target_hydroids)} streams between {ordr}-order streams are thalwag notch, including hydroIDs {target_hydroids}" + ) - # catchments_inund = catchments.copy() - # catchments_inund = np.where(np.isnan(rem_inund), np.nan, catchments_inund) + analysis = [ordr, target_hydroids, num_streams_order, cells_remaining, percentage_b0_rem0] - # catchments_inund_ls = catchments_inund[~np.isnan(catchments_inund)].tolist() - # hydroids_inund_ls = list(set(catchments_inund_ls)) - # hydroids_inund = [int(x) for x in hydroids_inund_ls] + analysis_data_ordr.append(analysis) - # hydroIds_branch0_noinund = len(hydroids_inund) - len(list(set(hydroids_inund) & set(hydroids_rem0))) + return analysis_data_ordr - # Simple Plotting - # reaches_gpkg = Path(branch0_dir, "demDerived_reaches_split_filtered_addedAttributes_crosswalked_0.gpkg") - # reaches = gpd.read_file(reaches_gpkg) - # plot_bounds = reaches.loc[reaches.HydroID.astype(int) == hydroid].bounds.iloc[0] - # Plot the zero REM cells - fig = plt.figure(figsize=(12, 7)) - ax = fig.subplots(1, 1) +# ------------------------------------------------------ +def analysis_missing_fim_cells(huc8_dir): - # reaches.plot(ax=ax, color='brown') - im = ax.imshow(rem_zeros, cmap='bwr', extent=rem_extent, interpolation='none') + list_subdir = os.listdir(huc8_dir) + dir_path_ls = [os.path.join(huc8_dir, subdir) for subdir in list_subdir] - path_fig = Path(dir_fig, 'rem_zeros.png') - plt.savefig(im, path_fig) + missing_fim_data = [] + for path1 in dir_path_ls: - """ + list_subdir = os.listdir(path1) + huc8 = [stg for stg in list_subdir if stg.isdigit()][0] + missing_fim = find_missing_fim_cells_for1huc(path1, huc8) - return [target_hydroids, cells_remaining, percentage_b0_rem0] + missing_fim_data.append([huc8, missing_fim]) + # Number of thalwag_notch_streams grouped by stream orders + thalwag_notch_ord = [] + for ord1 in range(6): + thalwag_notch = [] + for subls in missing_fim_data: + if len(subls[1]) >= (ord1 + 1): + thalwag_notch.append(len(subls[1][ord1][1])) + thalwag_notch_ord.append(sum(thalwag_notch)) + + # Number of streams grouped by stream orders + streams_ord = [] + for ord1 in range(6): + streams = [] + for subls in missing_fim_data: + if len(subls[1]) >= (ord1 + 1): + streams.append(subls[1][ord1][2]) + streams_ord.append(sum(streams)) + + return thalwag_notch_ord, streams_ord + + +# fim_dir = "/home/rdp-user/outputs/healded_fim_hca_01080107_if2/" +# fim_dir_str = Path(fim_dir) +# huc8_num = "01080107" +huc8_dir = "/efs-drives/fim-dev-efs/fim-home/heidi.safa/outputs/healed-fim-removing-hcas-analysis/" +thalwag_notch_ord, streams_ord = analysis_missing_fim_cells(huc8_dir) + +stream_orders = ["1st order", "2nd order", "3rd order", "4th order", "5th order", "6th order"] +percentage_notches = [ + round(100 * (thalwag_notch_ord[i] / streams_ord[i]), 4) for i in range(len(streams_ord)) +] + +path2savefig = ( + "/efs-drives/fim-dev-efs/fim-home/heidi.safa/outputs/path2savefig_hh_rhcas/Thalwag_Notch_Streams_perc.png" +) + +# creating the bar plot +fig = plt.figure(figsize=(20, 10)) +colors = [(135, 206, 255)] +plt.bar(stream_orders, percentage_notches, color=colors[0]) + +plt.xlabel("Stream Orders", fontsize=20) +plt.xticks(fontsize=20) +plt.ylabel("Percentage of Thalwag Notch Streams (%)", fontsize=18) +plt.yticks(fontsize=20) +plt.title("Percentage of catchments that never been Inundated", fontsize=20) +plt.show() +plt.savefig(path2savefig) if __name__ == "__main__": @@ -152,4 +189,4 @@ def find_missing_fim_cells(fim_dir_str, huc8_num, dir_fig): if not os.path.exists(dir_fig): os.mkdir(dir_fig) - find_missing_fim_cells(fim_dir, huc8_num, dir_fig) + analysis_missing_fim_cells(fim_dir, huc8_num, dir_fig) From 918fc6f792b3d4ec09fefbc2ff6294972dc91e1f Mon Sep 17 00:00:00 2001 From: hhs732 Date: Tue, 14 May 2024 20:11:54 +0000 Subject: [PATCH 5/7] analyze_missing_fim_cells.py upgraded --- tools/analyze_for_missing_FIM_cells.py | 85 ++++++++++++-------------- 1 file changed, 40 insertions(+), 45 deletions(-) diff --git a/tools/analyze_for_missing_FIM_cells.py b/tools/analyze_for_missing_FIM_cells.py index 766cae4b4..bbb23b7cd 100644 --- a/tools/analyze_for_missing_FIM_cells.py +++ b/tools/analyze_for_missing_FIM_cells.py @@ -1,5 +1,12 @@ """ -what is the purpose, example usage, link to the PR + +This tool analyzes the missing FIM cells after removing hydro-conditioning artifacts (thalweg rem). +It calculates the number of FIM celles in each HUC that will not be inundated after removing thalweg rem. +It also calculates the number of catchments that never shows any inundation because of the thalwag notch +issues, called notched streams. Then it creates a bar chart depicting the percentage of notched streams in +across all processed HUCs grouped by stream orders. + +You may find more detailes in PR #1156 (https://github.com/NOAA-OWP/inundation-mapping/pull/1156) """ @@ -64,7 +71,7 @@ def find_missing_fim_cells_for1huc(fim_dir, huc8_num): percentage_b0_rem0 = round(100 * (cells_remaining / non_zero_count), 4) print("") print( - f"Actual percentage of branch0, {ordr}-order stream cells that have not inundated: {percentage_b0_rem0}%" + f"Actual percentage of branch0, {ordr}-order stream cells that have not been inundated: {percentage_b0_rem0}%" ) # Finding branch0 hydroIDs that do not have zero rem (never inundate, notches) @@ -114,7 +121,8 @@ def analysis_missing_fim_cells(huc8_dir): thalwag_notch = [] for subls in missing_fim_data: if len(subls[1]) >= (ord1 + 1): - thalwag_notch.append(len(subls[1][ord1][1])) + if subls[1][ord1][0] == ord1 + 1: + thalwag_notch.append(len(subls[1][ord1][1])) thalwag_notch_ord.append(sum(thalwag_notch)) # Number of streams grouped by stream orders @@ -123,70 +131,57 @@ def analysis_missing_fim_cells(huc8_dir): streams = [] for subls in missing_fim_data: if len(subls[1]) >= (ord1 + 1): - streams.append(subls[1][ord1][2]) + if subls[1][ord1][0] == ord1 + 1: + streams.append(subls[1][ord1][2]) streams_ord.append(sum(streams)) return thalwag_notch_ord, streams_ord -# fim_dir = "/home/rdp-user/outputs/healded_fim_hca_01080107_if2/" -# fim_dir_str = Path(fim_dir) -# huc8_num = "01080107" -huc8_dir = "/efs-drives/fim-dev-efs/fim-home/heidi.safa/outputs/healed-fim-removing-hcas-analysis/" -thalwag_notch_ord, streams_ord = analysis_missing_fim_cells(huc8_dir) - -stream_orders = ["1st order", "2nd order", "3rd order", "4th order", "5th order", "6th order"] -percentage_notches = [ - round(100 * (thalwag_notch_ord[i] / streams_ord[i]), 4) for i in range(len(streams_ord)) -] - -path2savefig = ( - "/efs-drives/fim-dev-efs/fim-home/heidi.safa/outputs/path2savefig_hh_rhcas/Thalwag_Notch_Streams_perc.png" -) - -# creating the bar plot -fig = plt.figure(figsize=(20, 10)) -colors = [(135, 206, 255)] -plt.bar(stream_orders, percentage_notches, color=colors[0]) - -plt.xlabel("Stream Orders", fontsize=20) -plt.xticks(fontsize=20) -plt.ylabel("Percentage of Thalwag Notch Streams (%)", fontsize=18) -plt.yticks(fontsize=20) -plt.title("Percentage of catchments that never been Inundated", fontsize=20) -plt.show() -plt.savefig(path2savefig) - if __name__ == "__main__": # Parse arguments. parser = argparse.ArgumentParser(description="Analysis for missing FIM cells.") parser.add_argument( "-r", - dest="fim_dir_str", + dest="huc8s_dir", help="Path to directory storing FIM outputs. Type = string", required=True, type=str, ) parser.add_argument( - "-w", dest="huc8_num", help="HUC8 that is being analyzsd. Type = string", required=True, type=str - ) - parser.add_argument( - "-o", dest="dir_fig", help="Path to save rem_zero. Type = string", required=True, type=str + "-o", dest="path2savefig", help="Path to save bar chart. Type = string", required=True, type=str ) # Assign variables from arguments. args = vars(parser.parse_args()) - fim_dir = args["fim_dir_str"] - huc8_num = args["huc8_num"] - dir_fig = args["dir_fig"] + huc8s_dir = args["huc8s_dir"] + path2savefig = args["path2savefig"] - if not os.path.exists(fim_dir): - print("FIM directory: " + fim_dir + " does not exist.") + if not os.path.exists(huc8s_dir): + print("FIM directory: " + huc8s_dir + " does not exist.") quit - if not os.path.exists(dir_fig): - os.mkdir(dir_fig) + if not os.path.exists(path2savefig): + os.mkdir(path2savefig) + + thalwag_notch_ord, streams_ord = analysis_missing_fim_cells(huc8s_dir) + + stream_orders = ["1st order", "2nd order", "3rd order", "4th order", "5th order", "6th order"] + percentage_notches = [ + round(100 * (thalwag_notch_ord[i] / streams_ord[i]), 4) for i in range(len(streams_ord)) + ] + + path2savefig1 = os.path.join(path2savefig, "Thalwag_Notch_Streams_perc.png") + + # creating the bar plot + fig = plt.figure(figsize=(20, 10)) + plt.bar(stream_orders, percentage_notches) - analysis_missing_fim_cells(fim_dir, huc8_num, dir_fig) + plt.xlabel("Stream Orders", fontsize=20) + plt.xticks(fontsize=20) + plt.ylabel("Percentage of Thalwag Notch Streams (%)", fontsize=20) + plt.yticks(fontsize=20) + plt.title("Percentage of catchments that will not been iundated", fontsize=24) + plt.savefig(path2savefig1) From 9e50079b781aa7cea825a3ca8ac5a677e0558c2f Mon Sep 17 00:00:00 2001 From: HeidiSafa-NOAA Date: Fri, 17 May 2024 12:24:49 -0700 Subject: [PATCH 6/7] Update CHANGELOG.md --- docs/CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c904fef02..193e086ff 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,22 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. +## v4.5.x.x - 2024-05-16 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150) + +This focuses on removing hydro-conditioning artifacts by subtracting the thalweg DEM from HAND REM and adding back the original DEM. Also, a new tool was created to test this feature over multiple HUCs + +### Additions +- `tools/analyze_for_missing_FIM_cells.py`: A new script `analyze_for_missing_FIM_cells.py` was added to test and analyze healed HAND for hydro-conditioning artifacts FIM. + +### Changes +- `src/delineate_hydros_and_produce_HAND.sh`: Removing hydro-conditioning artifacts from HAND REM. +- `config/params_template.env`: Creating an option to include/exclude healed HAND from FIM pipeline. + +### Testing +This PR has been tested over 2 HUC8s including 12090301 and 01080107. + +

+ ## v4.5.0.1 - 2024-05-09 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150) Fixes two bugs discovered in v4.5.0.0: From 3f2d9103f8d8ec9928804220ba06cadef44401dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CRobHanna-NOAA=E2=80=9D?= <“Robert.Hanna@NOAA.gov”> Date: Fri, 17 May 2024 20:07:43 +0000 Subject: [PATCH 7/7] update changelog --- docs/CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index fbb44d13d..c4b27d9fb 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,7 +1,7 @@ All notable changes to this project will be documented in this file. We follow the [Semantic Versioning 2.0.0](http://semver.org/) format. -## v4.5.x.x - 2024-05-16 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150) +## v4.5.1.0 - 2024-05-17 - [PR#1150](https://github.com/NOAA-OWP/inundation-mapping/pull/1150) This focuses on removing hydro-conditioning artifacts by subtracting the thalweg DEM from HAND REM and adding back the original DEM. Also, a new tool was created to test this feature over multiple HUCs @@ -12,8 +12,6 @@ This focuses on removing hydro-conditioning artifacts by subtracting the thalweg - `src/delineate_hydros_and_produce_HAND.sh`: Removing hydro-conditioning artifacts from HAND REM. - `config/params_template.env`: Creating an option to include/exclude healed HAND from FIM pipeline. -### Testing -This PR has been tested over 2 HUC8s including 12090301 and 01080107.