[1pt] PR: Updating modules with new AEP (NWM V3)

docs/CHANGELOG.md

@@ -1,6 +1,15 @@
 All notable changes to this project will be documented in this file.
 We follow the [Semantic Versioning 2.0.0](http://semver.org/) format.
 
+## v4.x.x.x - 2024-06-28 - [PR#1198](https://github.com/NOAA-OWP/inundation-mapping/pull/1198)
+
+### Changes
+- `src/bash_variables.env`: high water threshold and recurrence flows CSV files were updated into new NWM v3 flow files. Also, a new Manning numbers file created from the new NWM v3 dataset was used.
+-  `src/src_adjust_ras2fim_rating.py`: 100 year recurrence was removed since it is not included in the new AEP.
+-  `src/src_adjust_usgs_rating_trace.py`: 100 year recurrence was removed since it is not included in the new AEP.
+-  `tools/rating_curve_comparison.py`: 100 year recurrence was removed since it is not included in the new AEP. Also, the name of recurrence flow CSV file was updated.
+-  `tools/composite_inundation.py`
+-  `tools/inundate_nation.py`
 
 ## v4.5.3.1 - 2024-07-24 - [PR#1233](https://github.com/NOAA-OWP/inundation-mapping/pull/1233)
 

src/bash_variables.env

@@ -35,13 +35,13 @@ export                        bathymetry_file=${inputsDir}/bathymetry/bathymetry
 export                            osm_bridges=${inputsDir}/osm/bridges/240426/osm_all_bridges.gpkg
 
 # input file location with nwm feature_id and recurrence flow values
-export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm_high_water_threshold_cms.csv
+export               bankfull_flows_file=${inputsDir}/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv
 
 # input file location with nwm feature_id and channel roughness and overbank roughness attributes
-export                  vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_06_12.csv
+export                  vmann_input_file=${inputsDir}/rating_curve/variable_roughness/mannings_global_nwm3.csv
 
 # input file location with nwm feature_id and recurrence flow values
-export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm21_17C_recurrence_flows_cfs.csv
+export                    nwm_recur_file=${inputsDir}/rating_curve/nwm_recur_flows/nwm3_17C_recurrence_flows_cfs.csv
 
 # input file location with usgs rating curve database
 export             usgs_rating_curve_csv=${inputsDir}/usgs_gages/usgs_rating_curves.csv

src/src_adjust_ras2fim_rating.py

@@ -89,29 +89,27 @@ def create_ras2fim_rating_database(ras_rc_filepath, ras_elev_df, nwm_recurr_file
 
     # read in the NWM recurr csv file
     nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
-    nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
+    if "Unnamed: 0" in nwm_recur_df.columns:
+        nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
     nwm_recur_df.rename(
         columns={
             '2_0_year_recurrence_flow_17C': '2_0_year',
             '5_0_year_recurrence_flow_17C': '5_0_year',
             '10_0_year_recurrence_flow_17C': '10_0_year',
             '25_0_year_recurrence_flow_17C': '25_0_year',
             '50_0_year_recurrence_flow_17C': '50_0_year',
-            '100_0_year_recurrence_flow_17C': '100_0_year',
         },
         inplace=True,
     )
 
     # convert cfs to cms (x 0.028317)
-    nwm_recur_df.loc[
-        :, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year', '100_0_year']
-    ] *= 0.028317
+    nwm_recur_df.loc[:, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year']] *= 0.028317
 
     # merge nwm recurr with ras_rc_df
     merge_df = ras_rc_df.merge(nwm_recur_df, how='left', on='feature_id')
 
     # NWM recurr intervals
-    recurr_intervals = ["2", "5", "10", "25", "50", "100"]  # "2","5","10","25","50","100"
+    recurr_intervals = ["2", "5", "10", "25", "50"]  # "2","5","10","25","50","100"
     final_df = pd.DataFrame()  # create empty dataframe to append flow interval dataframes
     for interval in recurr_intervals:
         log_text += '\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n'

src/src_adjust_usgs_rating_trace.py

@@ -87,28 +87,26 @@ def create_usgs_rating_database(usgs_rc_filepath, usgs_elev_df, nwm_recurr_filep
 
     # read in the NWM recurr csv file
     nwm_recur_df = pd.read_csv(nwm_recurr_filepath, dtype={'feature_id': int})
-    nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
+    if "Unnamed: 0" in nwm_recur_df.columns:
+        nwm_recur_df = nwm_recur_df.drop(columns=["Unnamed: 0"])
     nwm_recur_df = nwm_recur_df.rename(
         columns={
             '2_0_year_recurrence_flow_17C': '2_0_year',
             '5_0_year_recurrence_flow_17C': '5_0_year',
             '10_0_year_recurrence_flow_17C': '10_0_year',
             '25_0_year_recurrence_flow_17C': '25_0_year',
             '50_0_year_recurrence_flow_17C': '50_0_year',
-            '100_0_year_recurrence_flow_17C': '100_0_year',
         }
     )
 
     # convert cfs to cms (x 0.028317)
-    nwm_recur_df.loc[
-        :, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year', '100_0_year']
-    ] *= 0.028317
+    nwm_recur_df.loc[:, ['2_0_year', '5_0_year', '10_0_year', '25_0_year', '50_0_year']] *= 0.028317
 
     # merge nwm recurr with usgs_rc
     merge_df = usgs_rc_df.merge(nwm_recur_df, how='left', on='feature_id')
 
     # NWM recurr intervals
-    recurr_intervals = ("2", "5", "10", "25", "50", "100")
+    recurr_intervals = ("2", "5", "10", "25", "50")
     final_df = pd.DataFrame()  # create empty dataframe to append flow interval dataframes
     for interval in recurr_intervals:
         log_text += '\n\nProcessing: ' + str(interval) + '-year NWM recurr intervals\n'

tools/composite_inundation.py

@@ -433,7 +433,7 @@ def to_bin(x):
     python3 /foss_fim/tools/composite_inundation.py
         -fr /outputs/inundation_test_1_FIM3_fr
         -gms /outputs/inundation_test_1_gms
-        -u 13090001 -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_25_0_cms.csv
+        -u 13090001 -f /data/inputs/rating_curve/nwm_recur_flows/nwm3_17C_recurr_25_0_cms.csv
         -o /outputs/inundation_test_1_comp/
         -n test_inundation.tif
     """

tools/inundate_nation.py

@@ -16,7 +16,7 @@
 from utils.shared_functions import FIM_Helpers as fh
 
 
-# INUN_REVIEW_DIR = r'/data/inundation_review/inundation_nwm_recurr/'
+# INUN_REVIEW_DIR = r'/data/inputs/rating_curve/nwm_recur_flows/'
 # INUN_OUTPUT_DIR = r'/data/inundation_review/inundate_nation/'
 # INPUTS_DIR = r'/data/inputs'
 # OUTPUT_BOOL_PARENT_DIR = '/data/inundation_review/inundate_nation/bool_temp/
@@ -273,15 +273,15 @@ def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO
     Sample usage:
     python3 /foss_fim/tools/inundate_nation.py
         -r /outputs/fim_4_0_9_2 -m 100_0
-        -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm21_17C_recurr_100_0_cms.csv
+        -f /data/inputs/rating_curve/nwm_recur_flows/nwm3_17C_recurr_10_0_cms.csv
         -s
         -j 10
     outputs become /data/inundation_review/inundate_nation/100_0_fim_4_0_9_2_mosiac.tif (.log, etc)
 
     python3 /foss_fim/tools/inundate_nation.py
         -r /outputs/fim_4_0_9_2
         -m hw
-        -f /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/nwm_high_water_threshold_cms.csv
+        -f /data/inputs/rating_curve/bankfull_flows/nwm3_high_water_threshold_cms.csv
         -s
         -j 10
     outputs become /data/inundation_review/inundate_nation/hw_fim_4_0_9_2_mosiac.tif (.log, etc)
@@ -329,8 +329,8 @@ def __setup_logger(output_folder_path, log_file_name_key, log_level=logging.INFO
         '-f',
         '--flow_file',
         help='the path and flow file to be used. '
-        'ie /data/inundation_review/inundation_nwm_recurr/nwm_recurr_flow_data/'
-        'nwm_high_water_threshold_cms.csv',
+        'ie /data/inputs/rating_curve/bankfull_flows/'
+        'nwm3_high_water_threshold_cms.csv',
         required=True,
     )
 

tools/rating_curve_comparison.py

@@ -200,10 +200,10 @@ def generate_rating_curve_metrics(args):
             rating_curves['order_'] = rating_curves['order_'].astype('int')
 
             # NWM recurr intervals
-            recurr_intervals = ("2", "5", "10", "25", "50", "100")
+            recurr_intervals = ("2", "5", "10", "25", "50")
             recurr_dfs = []
             for interval in recurr_intervals:
-                recurr_file = join(nwm_flow_dir, 'nwm21_17C_recurr_{}_0_cms.csv'.format(interval))
+                recurr_file = join(nwm_flow_dir, 'nwm3_17C_recurr_{}_0_cms.csv'.format(interval))
                 df = pd.read_csv(recurr_file, dtype={'feature_id': str})
                 # Update column names
                 df = df.rename(columns={"discharge": interval})