diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index d4f4011f4..15704a4e9 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,6 +1,26 @@
All notable changes to this project will be documented in this file.
We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
+## v4.5.x.x - 2024-06-20 - [PR#1196](https://github.com/NOAA-OWP/inundation-mapping/pull/1196)
+
+This PR adds a new tool, which is still in progress. The tool addresses the issue #994 (but is not going to close this issue yet).
+The tool aims to update the HAND SRC by comparing water surface elevation (WSE) between RAS2FIM and HAND. The algorithm is as below:
+
+- For each RAS cross section, determine the HAND discharge adjustment (Q_Adjust):
+ - HAND minimum WSE is assumed the same as HAND Hydro-conditioned DEM
+ - RAS2FIM minimum WSE is already available from HEC-RAS runs.
+ - Calculate the difference between the minimum WSE values of RAS2FIM and HAND, and call it "WSE_base_error". This difference can address the bathymetry error in HAND SRC.
+ - Add stage values into RAS2FIM rating curves (by subtracting DEM from WSE), and then interpolate a discharge for the "WSE_base_error". This interpolated discharge is called "Q_Adjust."
+- Calculate the median of "Q_Adjust" values for cross sections within HAND catchments.
+- Add the median "Q_Adjust" values to the flows of HAND SRC
+
+
+
+### Additions
+- tools/adjust_wse_with_ras2fim.py
+
+
+
## v4.5.2.3 - 2024-06-14 - [PR#1169](https://github.com/NOAA-OWP/inundation-mapping/pull/1169)
This tool scans all log directory looking for the word "error" (not case-sensitive). This is primary added to help find errors in the post processing logs such as src_optimization folder (and others).
diff --git a/tools/adjust_wse_with_ras2fim.py b/tools/adjust_wse_with_ras2fim.py
new file mode 100644
index 000000000..ae670737d
--- /dev/null
+++ b/tools/adjust_wse_with_ras2fim.py
@@ -0,0 +1,343 @@
+import os
+from multiprocessing import Pool
+
+import geopandas as gpd
+import matplotlib.lines as mlines
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import rasterio
+from numpy import interp
+
+
+def get_ras2fim_discharge(args):
+ ras_rating_with_stage, this_WSE_base_error, this_point_fid_xs = args
+ this_ras_rating = ras_rating_with_stage[ras_rating_with_stage["fid_xs"] == this_point_fid_xs].sort_values(
+ by="flow"
+ )
+
+ all_flows = this_ras_rating["flow"].values
+ all_stage = this_ras_rating["stage"].values
+
+ if this_WSE_base_error > 0:
+ adjust_value = interp(this_WSE_base_error, all_stage, all_flows)
+ else:
+ adjust_value = 0
+
+ return this_point_fid_xs, adjust_value
+
+
+class adjust_wse_with_ras2fim:
+ '''
+ This code addresses the issue: https://github.com/NOAA-OWP/inundation-mapping/issues/994
+ The results of this work have been used here: https://docs.google.com/presentation/d/1OYqNWQKPgYquWxnJgYcrvuV67iWzwikZ/edit#slide=id.p1
+ '''
+
+ def __init__(self, ras_dir, HAND_dir, hand_discharge_type, output_dir):
+ self.ras_dir = ras_dir
+ self.HAND_dir = HAND_dir
+ self.hand_discharge_type = hand_discharge_type
+ self.output_dir = output_dir
+
+ def get_hydroIds_QAdjust(self, ras_points_with_rc, ras_rating, HAND_SRC):
+ '''
+ For the details and the algorithm, see this issue: https://github.com/NOAA-OWP/inundation-mapping/issues/994
+ '''
+
+ def get_ras2fim_discharge_parallel(ras_points_with_rc, ras_rating_with_stage):
+ args = []
+ for i, row in ras_points_with_rc.iterrows():
+ args.append((ras_rating_with_stage, row.WSE_base_error, row.fid_xs))
+
+ with Pool(7) as pool:
+
+ list_of_returned_results = pool.map(get_ras2fim_discharge, args)
+
+ ras2fim_Qs = pd.DataFrame(list_of_returned_results, columns=["fid_xs", "Q_Adjust_temp"])
+ return ras2fim_Qs
+
+ # the min WSE of ras is needed to compare with HAND min WSE
+ min_flow_idx = ras_rating.groupby('fid_xs')['flow'].idxmin() # use for min flow
+
+ # Use the row indices to extract the corresponding 'WSE' values.
+ min_flow_wse = ras_rating.loc[min_flow_idx, ['fid_xs', 'flow', 'wse']]
+ min_flow_wse.columns = ['fid_xs', 'ras_min_flow', 'ras_min_wse']
+
+ # merge the results into ras_points_with_rc
+ ras_points_with_rc = ras_points_with_rc.merge(min_flow_wse, on='fid_xs')
+
+ # now start to compute min WSE of HAND
+ # TODO examine using WSE of actual minimum flow instead of assuming the stage to be zero
+ # Note... I tested above and the differenc (improvment in results) ewas insignificant.
+ # get DEM values for ras points
+ DEM_dataset = rasterio.open(os.path.join(self.HAND_dir, 'dem_thalwegCond_0.tif'))
+ ras_points_coords = list(
+ zip(ras_points_with_rc.geometry.centroid.x, ras_points_with_rc.geometry.centroid.y)
+ )
+ ras_points_with_rc["DEM_datum"] = [value[0] for value in DEM_dataset.sample(ras_points_coords)]
+
+ # therefore, HAND minimum WSE is equl to DEM + first stage (this is Ryan method that we call this script after)
+ ras_points_with_rc["HAND_min_wse"] = ras_points_with_rc["DEM_datum"].values + 0.3048
+
+ # compute the base error
+ ras_points_with_rc["WSE_base_error"] = (
+ ras_points_with_rc["HAND_min_wse"] - ras_points_with_rc["ras_min_wse"]
+ )
+
+ # convert ras2fim rating WSE to stage
+ ras_rating_with_stage = ras_rating.merge(ras_points_with_rc[["fid_xs", "DEM_datum"]], on="fid_xs")
+ ras_rating_with_stage["stage"] = ras_rating_with_stage["wse"] - ras_rating_with_stage["DEM_datum"]
+
+ # Interpolate a discharge from ras2fim discharge-stage rating curves for the "WSE_base_error", and call it "Q_Adjust"
+ ras2fim_Qs = get_ras2fim_discharge_parallel(ras_points_with_rc, ras_rating_with_stage)
+ ras_points_with_rc = ras_points_with_rc.merge(ras2fim_Qs, on="fid_xs")
+
+ # Compute the median of "Q_Adjust" of ras2fim points within each HAND HydroID
+ # to do that, first we need to get HydroID for ras points
+ HydroID_tif_dataset = rasterio.open(
+ os.path.join(self.HAND_dir, "gw_catchments_reaches_filtered_addedAttributes_0.tif")
+ )
+ ras_points_with_rc["HydroID"] = [value[0] for value in HydroID_tif_dataset.sample(ras_points_coords)]
+
+ # now retrieve the first available Q from HAND SRC
+ # as a temp dataset needed, remove stage=0 from HAND SRC so we can apply .min to get first discharge
+ # only select HydroIds needed for ras_point
+ HAND_SRC = HAND_SRC[HAND_SRC['HydroID'].isin(ras_points_with_rc["HydroID"].unique())]
+
+ HAND_SRC_temp = HAND_SRC[HAND_SRC['stage'] != 0]
+
+ HAND_first_q = HAND_SRC_temp.groupby("HydroID")[self.hand_discharge_type].min().reset_index()
+ HAND_first_q.rename(columns={self.hand_discharge_type: "HAND_first_q"}, inplace=True)
+
+ ras_points_with_rc = ras_points_with_rc.merge(HAND_first_q, on='HydroID')
+
+ ras_points_with_rc["Q_Adjust"] = (
+ ras_points_with_rc["Q_Adjust_temp"] - ras_points_with_rc["HAND_first_q"]
+ )
+
+ # compute median of baseline error Q for the points within each hydroid
+ HydroIDs_QAdjust = ras_points_with_rc.groupby("HydroID")["Q_Adjust"].median()
+ ras_points_with_rc['Q_HydroId_Median'] = ras_points_with_rc['HydroID'].map(HydroIDs_QAdjust)
+
+ # report the required adusted HAND flows
+ return ras_points_with_rc, ras_points_with_rc[['HydroID', 'Q_HydroId_Median']].drop_duplicates(
+ subset='HydroID'
+ )
+
+ def compare_ras_with_hand(self, ras_points_with_rc, ras_rating, updated_HAND_SRC, flow_intensity):
+ '''
+ This method compute the difference of WSE between HAND and ras2fim for four different ras2fim flow intensities of q1 (first quantile),
+ median, q3 (third quantile) and max.
+ The method assumes the existence of updated HAND SRC created in previous step.
+ '''
+
+ # find selected method of WSE from ras
+ ras_rating["orig_index"] = ras_rating.index.values
+ if flow_intensity == 'max':
+ method_flow = ras_rating.groupby('fid_xs')['flow'].max().reset_index()
+ elif flow_intensity == 'median':
+ method_flow = ras_rating.groupby('fid_xs')['flow'].median().reset_index()
+ elif flow_intensity == 'q1':
+ method_flow = (
+ ras_rating.groupby('fid_xs')['flow'].quantile(0.25, interpolation='nearest').reset_index()
+ )
+ elif flow_intensity == 'q3':
+ method_flow = (
+ ras_rating.groupby('fid_xs')['flow'].quantile(0.75, interpolation='nearest').reset_index()
+ )
+
+ # Merge the flows with the original DataFrame to get the corresponding indices
+ merged_df = ras_rating.merge(method_flow, on=['fid_xs', 'flow'], how='inner')
+
+ # Get the indices of the method flow rows
+ method_flow_idx = merged_df.set_index('fid_xs')['orig_index']
+
+ # Use the row indices to extract the corresponding 'WSE' values. Also get max flow from here and not "max_flow" of gpkg file
+ method_flow_wse = ras_rating.loc[method_flow_idx, ['fid_xs', 'flow', 'wse']]
+ method_flow_wse.columns = ['fid_xs', 'ras_%s_flow' % flow_intensity, 'ras_%s_wse' % flow_intensity]
+
+ # merge the results into ras_points_with_rc
+ ras_points_with_rc = ras_points_with_rc.merge(method_flow_wse, on='fid_xs')
+
+ # now get results at points using HAND
+ adjusted_HAND_stage = []
+ original_HAND_stage = []
+ for i, row in ras_points_with_rc.iterrows():
+ this_point_method_flow, this_point_HydroID = row["ras_%s_flow" % flow_intensity], row.HydroID
+ this_SRC = updated_HAND_SRC[updated_HAND_SRC["HydroID"] == this_point_HydroID].sort_values(
+ by=self.hand_discharge_type
+ )
+
+ all_flows_adjust = this_SRC[self.hand_discharge_type].values
+ all_flows_orig = this_SRC['original_' + self.hand_discharge_type].values
+ all_stages = this_SRC["stage"].values
+
+ # Perform linear interpolation
+ adjusted_HAND_stage.append(interp(this_point_method_flow, all_flows_adjust, all_stages))
+ original_HAND_stage.append(interp(this_point_method_flow, all_flows_orig, all_stages))
+
+ ras_points_with_rc["Adjusted_HAND_%s_stage" % flow_intensity] = np.array(adjusted_HAND_stage)
+ ras_points_with_rc["Original_HAND_%s_stage" % flow_intensity] = np.array(original_HAND_stage)
+
+ ras_points_with_rc["Adjusted_HAND_%s_wse" % flow_intensity] = (
+ ras_points_with_rc["Adjusted_HAND_%s_stage" % flow_intensity] + ras_points_with_rc["DEM_datum"]
+ )
+ ras_points_with_rc["Original_HAND_%s_wse" % flow_intensity] = (
+ ras_points_with_rc["Original_HAND_%s_stage" % flow_intensity] + ras_points_with_rc["DEM_datum"]
+ )
+
+ ras_points_with_rc["Adjusted_HAND_%s_wse" % flow_intensity] = np.where(
+ ras_points_with_rc["Original_HAND_%s_wse" % flow_intensity].values
+ > ras_points_with_rc['ras_%s_wse' % flow_intensity].values,
+ ras_points_with_rc["Adjusted_HAND_%s_wse" % flow_intensity].values,
+ ras_points_with_rc["Original_HAND_%s_wse" % flow_intensity].values,
+ )
+
+ ras_points_with_rc["Orig_Error"] = (
+ ras_points_with_rc["ras_%s_wse" % flow_intensity]
+ - ras_points_with_rc["Original_HAND_%s_wse" % flow_intensity]
+ )
+ ras_points_with_rc["Adj_Error"] = (
+ ras_points_with_rc["ras_%s_wse" % flow_intensity]
+ - ras_points_with_rc["Adjusted_HAND_%s_wse" % flow_intensity]
+ )
+
+ ras_points_with_rc.drop(
+ columns=["Adjusted_HAND_%s_stage" % flow_intensity, "Original_HAND_%s_stage" % flow_intensity],
+ inplace=True,
+ )
+
+ return ras_points_with_rc
+
+ def comparison_plots(self, ras_points_Adjusted_Q, flow_intensity):
+ '''
+ This method make difference plots using both original HAND SRC and updated HAND SRC.
+ '''
+ if not os.path.exists(os.path.join(self.output_dir, 'plots')):
+ os.makedirs(os.path.join(self.output_dir, 'plots'))
+
+ for error_type in ['Orig', 'Adj']:
+ ras_points_Adjusted_Q["residual_color"] = np.where(
+ ras_points_Adjusted_Q["%s_Error" % error_type].values > 0, "red", "blue"
+ )
+
+ RMSE = np.sqrt(np.mean(ras_points_Adjusted_Q["%s_Error" % error_type].values ** 2))
+ MAE = np.mean(abs(ras_points_Adjusted_Q["%s_Error" % error_type].values))
+ ras_points_Adjusted_Q.plot(color=ras_points_Adjusted_Q["residual_color"].values, markersize=4)
+
+ # Create custom legend markers (two points)
+ legend_marker2 = mlines.Line2D(
+ [], [], color='blue', marker='o', linestyle='', markersize=4, label='HAND > RAS'
+ )
+ legend_marker1 = mlines.Line2D(
+ [], [], color='red', marker='o', linestyle='', markersize=4, label='RAS > HAND'
+ )
+
+ # Add the custom legend markers to the legend
+ plt.legend(handles=[legend_marker1, legend_marker2])
+ plt.title(
+ "WSE comparison between RAS2FIM and %s HAND for %s flows\nRMSE=%.2lfm, MAE=%.2lfm"
+ % (error_type, flow_intensity, RMSE, MAE)
+ )
+ plt.axis('off')
+
+ plt.savefig(
+ os.path.join(self.output_dir, 'plots', 'plot1_%s_%s.png' % (error_type, flow_intensity))
+ )
+ plt.close()
+
+ plt.axhline(y=0, color="blue", linestyle='--')
+ plt.scatter(
+ ras_points_Adjusted_Q["ras_%s_wse" % flow_intensity].values,
+ ras_points_Adjusted_Q["%s_Error" % error_type].values,
+ color="red",
+ s=4,
+ )
+
+ plt.xlabel("RAS2FIM WSE (m)")
+ plt.ylabel("RAS2FIM WSE minus HAND WSE (m)")
+ plt.title("For RAS2FIM %s flows using %s HAND SRC" % (flow_intensity, error_type))
+ plt.savefig(
+ os.path.join(self.output_dir, 'plots', 'plot2_%s_%s.png' % (error_type, flow_intensity))
+ )
+ plt.close()
+
+
+if __name__ == '__main__':
+ # specify required inputs
+ ras_dir = r"../../output_ras2fim_12090301"
+ HAND_dir = r"../../Ali_12090301\12090301\branches\0"
+ output_dir = r"../results/final"
+ hand_discharge_type = 'precalb_discharge_cms' # options are 'discharge_cms' or 'precalb_discharge_cms'
+ obs = adjust_wse_with_ras2fim(ras_dir, HAND_dir, hand_discharge_type, output_dir)
+
+ # read inputs
+ print('Reading input files')
+ ras_points = gpd.read_file(os.path.join(ras_dir, "reformat_ras_rating_curve_points.gpkg"))
+ ras_points = ras_points[['fid_xs', 'feature_id', 'geometry']]
+
+ # read rating curve info
+ ras_rating = pd.read_csv(
+ os.path.join(ras_dir, 'reformat_ras_rating_curve_table-12090301.csv'),
+ usecols=['fid_xs', 'wse', 'flow'],
+ )
+
+ # remove the points not having info in rating curve file
+ ras_points_with_rc = ras_points[ras_points["fid_xs"].isin(ras_rating["fid_xs"].unique())]
+
+ HAND_SRC = pd.read_csv(
+ os.path.join(HAND_dir, 'hydroTable_0.csv'), usecols=['HydroID', 'stage', hand_discharge_type]
+ )
+
+ # first get the hydroIds Q adjust
+ print('Calculating HAND HydroIDs Q adjust')
+ ras_points_Adjusted_Q, HydroIDs_Adjusted_Q = obs.get_hydroIds_QAdjust(
+ ras_points_with_rc, ras_rating, HAND_SRC
+ )
+ ras_points_Adjusted_Q.to_file(os.path.join(output_dir, "ras_points_Adjusted_Q.gpkg"))
+ HydroIDs_Adjusted_Q.to_csv(os.path.join(output_dir, "HydroIDs_Adjusted_Q.csv"), index=False)
+
+ # step 2...Adjust HAND SRCs
+ print('Updating HAND SRC with Q adjust values')
+ # TODO see why we have negative Q ajust for some hydroIDs...for now make them zero
+ HydroIDs_Adjusted_Q = HydroIDs_Adjusted_Q[HydroIDs_Adjusted_Q['Q_HydroId_Median'] > 0]
+
+ Orig_HAND_SRC = HAND_SRC.copy()
+ Orig_HAND_SRC.rename(columns={hand_discharge_type: 'original_%s' % hand_discharge_type}, inplace=True)
+
+ updated_HAND_SRC = Orig_HAND_SRC.merge(HydroIDs_Adjusted_Q, on='HydroID', how='left')
+
+ # assign zero for the hydroids that do not have any ras points
+ updated_HAND_SRC['Q_HydroId_Median'].fillna(0, inplace=True)
+
+ updated_HAND_SRC[hand_discharge_type] = (
+ updated_HAND_SRC['original_%s' % hand_discharge_type] + updated_HAND_SRC['Q_HydroId_Median']
+ )
+ updated_HAND_SRC.to_csv(os.path.join(output_dir, 'hydroTable_0_updated.csv'), index=False)
+
+ # finally compare RAS with HAND... this prepare difference between HAND and RAS2fim for both original HAND SRC and updated HAND SRC.
+ print('Evaluating the difference between HAND and SRC for different flow intensities:')
+ ras_points_Adjusted_Q = ras_points_Adjusted_Q[
+ ['fid_xs', 'feature_id', 'DEM_datum', 'HydroID', 'Q_HydroId_Median', 'geometry']
+ ]
+
+ MAE_summary = []
+ for flow_intensity in ['q1', 'median', 'q3', 'max']:
+ print('working on %s' % flow_intensity)
+ ras_points_Adjusted_Q = obs.compare_ras_with_hand(
+ ras_points_Adjusted_Q, ras_rating, updated_HAND_SRC, flow_intensity
+ )
+ ras_points_Adjusted_Q.to_file(os.path.join(output_dir, "RAS_HAND_Diff_%s.gpkg" % flow_intensity))
+ obs.comparison_plots(ras_points_Adjusted_Q, flow_intensity)
+
+ # also record MAE
+ MAE_orig_error = np.mean(abs(ras_points_Adjusted_Q['Orig_Error'].values))
+ MAE_Adj_error = np.mean(abs(ras_points_Adjusted_Q['Adj_Error'].values))
+
+ MAE_improve = 100 * (MAE_orig_error - MAE_Adj_error) / MAE_orig_error
+ MAE_summary.append([flow_intensity, MAE_orig_error, MAE_Adj_error, MAE_improve])
+ MAE_summary_DF = pd.DataFrame(
+ MAE_summary,
+ columns=["flow_intensity", 'MAE_difference_original', 'MAE_difference_updated', 'improvment%'],
+ )
+ MAE_summary_DF.to_csv(os.path.join(output_dir, 'MAE_stats.csv'), index=False)